Flakey Wallaby test: `execute_script()` followed by assertion

TLDR; In Wallaby, how do I correctly use execute_script() before an assertion? Does execute_script(_, _, _, assertion_callback) ensure synchronous code execution? But execute_script/2 does not?

I am trying to test an infinity scrolling implementation that uses LiveView’s streams with Wallaby.

My original test looked like this.

  @chat_window Query.data("role", "chat-window")
  @n_posts 40
  test "user scrolls up to see past posts", %{session: session} do
    posts = many_posts(@n_posts)
    topic = insert!(:topic, name: "General", posts: posts)
    groups = [insert!(:welcome_group, topics: [topic])]
    user = insert!(:user, groups: groups)

    session
    |> log_in_user(user)
    |> visit(~p"/groups")
    |> find(@chat_window)

    |> assert_in_viewport("[data-role=\"post\"]:first-child")

    |> scroll_chat_up_a_bit()
    |> refute_in_viewport("[data-role=\"post\"]:first-child")

    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> scroll_chat_up_a_bit()
    |> assert_in_viewport("[data-role=\"post\"]:nth-child(#{@n_posts})")
  end

  defp assert_in_viewport(parent, selector) do
    execute_script(parent,
      """
      const post = document.querySelector(arguments[0]);

      if (!post) return false;

      const isInViewport = (element) => {
        const rect = element.getBoundingClientRect();
        return (
            rect.top >= 0 &&
            rect.left >= 0 &&
            rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
            rect.right <= (window.innerWidth || document.documentElement.clientWidth)
        )
      }

      return isInViewport(post);
      """,
      [selector],
      fn resp -> assert resp == true end
    )
  end

  defp refute_in_viewport(parent, selector) do
    execute_script(parent,
      """
        const post = document.querySelector(arguments[0]);

        const isInViewport = (element) => {
          const rect = element.getBoundingClientRect();
          return (
              rect.top >= 0 &&
              rect.left >= 0 &&
              rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
              rect.right <= (window.innerWidth || document.documentElement.clientWidth)
          )
        }

        return isInViewport(post);
      """,
      [selector],
      fn resp -> assert resp == false end
    )
  end

  defp scroll_chat_up_a_bit(parent) do
    execute_script(parent,
      """
      const chat = document.querySelector('[data-role="chat-window"');
      chat.scrollBy(0, -400);
      """
    )
  end

  defp many_posts(amount) do
    Enum.reduce(1..amount, [], fn n, acc ->
      [insert!(:post, content: "Post #{n}") | acc ]
    end)
  end

This results in unreliable test results. Most of the time the test fails. Sometimes it doesn’t. If I add some :timer.sleep/1 time, the test passes consistently.

I reckoned the problem with my original test was code execution order. I adjusted my test to make use of the optional callback argument of the execute_script\4 function. This is the new test.

  @chat_window Query.data("role", "chat-window")
  @n_posts
  test "user scrolls up to see past posts", %{session: session} do
    posts = many_posts(@n_posts)
    topic = insert!(:topic, name: "General", posts: posts)
    groups = [insert!(:welcome_group, topics: [topic])]
    user = insert!(:user, groups: groups)

    session
    |> log_in_user(user)
    |> visit(~p"/groups")
    |> find(@chat_window)
    |> assert_in_viewport("[data-role=\"post\"]:first-child")

    session
    |> scroll_chat_up_a_bit(1, fn s ->
     refute_in_viewport(s, "[data-role=\"post\"]:first-child")
    end)

    session
    |> scroll_chat_up_a_bit(12, fn s ->
      assert_in_viewport(s, "[data-role=\"post\"]:nth-child(#{@n_posts})")
    end)
  end

  defp assert_in_viewport(parent, selector) do
    execute_script(parent,
      """
      const post = document.querySelector(arguments[0]);

      if (!post) return false;

      const isInViewport = (element) => {
        const rect = element.getBoundingClientRect();
        return (
            rect.top >= 0 &&
            rect.left >= 0 &&
            rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
            rect.right <= (window.innerWidth || document.documentElement.clientWidth)
        )
      }

      return isInViewport(post);
      """,
      [selector],
      fn resp -> assert resp == true end
    )
  end

  defp refute_in_viewport(parent, selector) do
    execute_script(parent,
      """
        const post = document.querySelector(arguments[0]);

        if (!post) throw new Error("Bottom post not found");

        const isInViewport = (element) => {
          const rect = element.getBoundingClientRect();
          return (
              rect.top >= 0 &&
              rect.left >= 0 &&
              rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
              rect.right <= (window.innerWidth || document.documentElement.clientWidth)
          )
        }

        return isInViewport(post);
      """,
      [selector],
      fn resp -> assert resp == false end
    )
  end

  defp scroll_chat_up_a_bit(parent, 0, callback), do: callback.(parent)

  defp scroll_chat_up_a_bit(parent, n, callback) when n > 0 do
    execute_script(parent,
      """
      const chat = document.querySelector('[data-role="chat-window"');
      chat.scrollBy(0, -400);
      """,
      [],
      fn _ -> scroll_chat_up_a_bit(parent, n - 1, callback) end
    )
  end

I was surprised to find that the second test also returns inconsistent test results.

My priority is to understand what is going wrong here. Any ideas?

Secondarily, it could very well be that I am over complicating this test. Better approaches are more than welcome.

P.S. There is a small section about asynchronous JavaScript in hexdocs of Wallaby, but following its advice did not solve my problem.