Ecto error: Getting "invalid keyword list in query" but can't locate issue

I keep getting this Ecto error and I’m completely stumped. I’ve looked at the documentation on keyword lists and I’ve tried process of elimination, but I just can’t figure out what is throwing this error. My keyword lists seem to be used at the top level positions in all of my “where” clauses.

The multi_reposition code is just pulled from Christ McCord’s Todo Trek example. Here’s the error and code. Is there an Ecto expert who could help me figure out what is creating the issue?

ERROR

[error] GenServer #PID<0.42094.0> terminating
** (Ecto.QueryError) lib/my_app/list_positions.ex:39: invalid keyword list in query:

from l0 in MyApp.LessonItems.LessonItem,
  where: l0.lesson_id == ^1,
  where: not (^[{:id, 10}]),
  where: l0.position >
  subquery(
    #Ecto.Query<from l0 in MyApp.LessonItems.LessonItem, where: l0.id == ^10, select: %{position: l0.position}>
  ) and l0.position <= ^3

keyword lists are only allowed at the top level of where, having, distinct, order_by, update or a join's on

    (elixir 1.15.4) lib/enum.ex:2510: Enum."-reduce/3-lists^foldl/2-0-"/3
    (elixir 1.15.4) lib/enum.ex:1819: Enum."-map_reduce/3-lists^mapfoldl/2-0-"/3
    (elixir 1.15.4) lib/enum.ex:1819: Enum."-map_reduce/3-lists^mapfoldl/2-0-"/3
    (elixir 1.15.4) lib/enum.ex:2510: Enum."-reduce/3-lists^foldl/2-0-"/3
    (ecto 3.10.3) lib/ecto/repo/queryable.ex:211: Ecto.Repo.Queryable.execute/4
    (ecto 3.10.3) lib/ecto/multi.ex:888: anonymous fn/5 in Ecto.Multi.operation_fun/2
    (ecto 3.10.3) lib/ecto/multi.ex:844: Ecto.Multi.apply_operation/5
    (elixir 1.15.4) lib/enum.ex:2510: Enum."-reduce/3-lists^foldl/2-0-"/3
    (ecto 3.10.3) lib/ecto/multi.ex:818: anonymous fn/5 in Ecto.Multi.apply_operations/5
    (ecto_sql 3.10.2) lib/ecto/adapters/sql.ex:1352: anonymous fn/3 in Ecto.Adapters.SQL.checkout_or_transaction/4
Last message: %Phoenix.Socket.Message{topic: "lv:phx-F82nEVZjxy9m1tSi", event: "event", payload: %{"event" => "reposition", "type" => "hook", "value" => %{"id" => "10", "new" => 3, "old" => 4, "phxStream" => "0"}}, ref: "8", join_ref: "4"}

CALLING FUNCTION TO MULTIPOSITION FUNCTION THAT HAS ERROR
This is the function sending the parameters. The last two parameters in the ListPositions function call are the keyword lists used in queries

  def update_lesson_item_position(_user_id, %Lesson{} = lesson, %LessonItem{} = lesson_item, new_idx)
      when is_map(lesson) and is_map(lesson_item) and lesson.id == lesson_item.lesson_id do
    Ecto.Multi.new()
    # lock is done in multi_reposition
    |> ListPositions.multi_reposition(:new, # name
                        lesson_item, # lesson_item to be reordered
                        {LessonItem, lesson.id}, # lock all lesson_items with lesson.id
                        new_idx, # new_idx
                        [lesson_id: lesson.id], # get_list_criteria (all lesson_items with lesson.id
                        [id: lesson_item.id]) # get_lesson_item_criteria for one being repositioned
    |> Repo.transaction()
    |> case do
      {:ok, _} ->
        # updated lesson_item with it's new position
        updated_lesson_item = %LessonItem{lesson_item | position: new_idx}
        # return in case LiveView wants to update stream
        {:ok, updated_lesson_item}

      {:error, failed_op, failed_val, _changes_so_far} ->
        Logger.error "Lessons.update_lesson_position failed_op: #{failed_op} changeset: #{inspect failed_val}"
        {:error, "Item could not be moved. Please try again."}
    end
  end

MULTI POSITION FUNCTION THAT IS NEARLY IDENTICAL TO CHRIS MCCORD’S CODE

 def multi_reposition(%Ecto.Multi{} = multi, name, %type{} = _struct, lock, new_idx, get_list_criteria, get_struct_criteria)
    when is_integer(new_idx) do

    old_position = from(og in type, where: ^get_struct_criteria, select: og.position)

    IO.puts "OLD POSITION: #{inspect old_position}"
    IO.puts "NAME: #{name} TYPE: #{inspect type} LOCK: #{inspect lock} NEW IDX: #{inspect new_idx}"
    IO.puts "LIST CRITERIA: #{inspect get_list_criteria}"
    IO.puts "STRUCT CRITERIA: #{inspect get_struct_criteria}"

    multi
    |> Repo.multi_transaction_lock(name, lock)
    |> Ecto.Multi.run({:index, name}, fn repo, _changes ->
      # retrieve the list and determine if item is going at end of list;
        case repo.one(from(t in type, where: ^get_list_criteria, select: count(t.position))) do # get count of list items
          count when new_idx < count -> {:ok, new_idx} # use new_index if not at end of list
          count -> {:ok, count - 1} # set index to count-1 if at end of list (0 based index)
        end
    end)
    |> multi_update_all({:dec_positions, name}, fn %{{:index, ^name} => computed_index} ->
# THIS IS LINE 39 WHERE ERROR message points to
      from(t in type,
        where: ^get_list_criteria, # get all items in list
        where: not ^get_struct_criteria, # except struct being moved
        # and where item.position > old position and <= new position
        where: t.position > subquery(old_position) and t.position <= ^computed_index,
        # decrement positions because shifting these up the list
        update: [inc: [position: -1]]
      )
    end)
    |> multi_update_all({:inc_positions, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
        where: ^get_list_criteria, # get all items in list
        where: not ^get_struct_criteria, # except struct being moved
        # and where item.position < old position and >= new position
        where: t.position < subquery(old_position) and t.position >= ^computed_index,
        # increment positions because shifting these down the list
        update: [inc: [position: 1]]
      )
    end)
    |> multi_update_all({:position, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
        # get the item being moved
        where: ^get_struct_criteria,
        # assign it new position
        update: [set: [position: ^computed_index]]
      )
    end)
  end

Here, [{:id, 10}] is the de-sugared version of [id: 10]. And thats not the top level where clause. Hence I suspect that is the source of the error.

iex> [{:id, 10}] == [id: 10]
true

That’s definitely it. I don’t remember how and if you can do with the bindingless syntax—I always just use the binding:

where: l0.id != 10

Thank you so much @kip and @sodapopcan! That particular keyword list was the issue. I tried moving it to the top of the query, but it still didn’t work. So I followed @sodapopcan’s advice and came up with a solution where I could explicitly specify the bindings and not use keyword lists.

Part of my issue is that I need to build queries that work with both single-key tables and double-key tables (my join tables). This meant that my keyword lists sometimes have two values: [primary_key_id1: value1, primary_key_id_2: value2] Trying to send two ids in the keyword list and have them generate accurate bindings was not working. Even just passing one key wasn’t working! It specifically fell over when I tried to use “not”:
where: not ^keyword_list. That was where the binding didn’t resolve correctly, regardless of whether I was sending one or two values.

In case others run into this problem, I’m including my final solution below. I have no doubt that there is a more elegant way to achieve this. I really struggle with Ecto. But the good news is that it works!

Again … this is an adaptation of the multi_reposition function that @chrismccord presented in his Todo Trek example. That example has been a great way to learn more about Multi transactions! This version works with tables that have more than one primary key.

I now just pass the field and it’s value as a tuple instead of a keyword list. Yes, I know … it’s basically a keyword list. But this works and passing them as keyword lists didn’t work. Go figure.

I then use ``Query.API `field(source, field)``` to dynamically insert the field into the query.

  #******************************************************************
  # multi_reposition where movable items have two PRIMARY KEYS (join table)
  #******************************************************************
  def multi_reposition(%Ecto.Multi{} = multi, name, %type{} = _struct, lock, new_idx,
                      {list_key_field, list_key_value}, {struct_key1_field, struct_key1_value},
                      {struct_key2_field, struct_key2_value}) do

    old_position_query =
      from(t in type, where: field(t, ^struct_key1_field) == ^struct_key1_value
      and field(t, ^struct_key2_field) == ^struct_key2_value, select: t.position)
    list_count_query =
      from(t in type, where: field(t, ^list_key_field) == ^list_key_value, select: count(t.position))

    multi
    |> Repo.multi_transaction_lock(name, lock)
    |> Ecto.Multi.run({:index, name}, fn repo, _changes ->
      # retrieve the list and determine if item is going at end of list;
        case repo.one(list_count_query) do # get count of list items
          count when new_idx < count -> {:ok, new_idx} # use new_index if not at end of list
          count -> {:ok, count - 1} # set index to count-1 if at end of list (0 based index)
        end
    end)
    |> multi_update_all({:dec_positions, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
      where: field(t, ^list_key_field) == ^list_key_value, # get all items in list
      where: field(t, ^struct_key1_field) != ^struct_key1_value # except struct being moved
      and    field(t, ^struct_key2_field) != ^struct_key2_value,
      # and where item.position > old position and <= new position
      where: t.position > subquery(old_position_query) and t.position <= ^computed_index,
      update: [inc: [position: -1]] # decrement positions because shifting these up the list
      )
    end)
    |> multi_update_all({:inc_positions, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
      where: field(t, ^list_key_field) == ^list_key_value, # get all items in list
      where: field(t, ^struct_key1_field) != ^struct_key1_value # except struct being moved
      and    field(t, ^struct_key2_field) != ^struct_key2_value,
      # and where item.position < old position and >= new position
      where: t.position < subquery(old_position_query) and t.position >= ^computed_index,
      update: [inc: [position: 1]] # increment positions because shifting these down the list
      )
    end)
    |> multi_update_all({:position, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
        where: field(t, ^struct_key1_field) == ^struct_key1_value # get item being moved
        and    field(t, ^struct_key2_field) == ^struct_key2_value,
        update: [set: [position: ^computed_index]] # assign it new position
      )
    end)
  end

  #******************************************************************
  # multi_reposition where movable item has one PRIMARY KEY
  #******************************************************************
  def multi_reposition(%Ecto.Multi{} = multi, name, %type{} = _struct, lock, new_idx,
                      {list_key_field, list_key_value}, {struct_key_field, struct_key_value}) do

    old_position_query =
      from(t in type, where: field(t, ^struct_key_field) == ^struct_key_value, select: t.position)
    list_count_query =
      from(t in type, where: field(t, ^list_key_field) == ^list_key_value, select: count(t.position))

    multi
    |> Repo.multi_transaction_lock(name, lock)
    |> Ecto.Multi.run({:index, name}, fn repo, _changes ->
      # retrieve the list and determine if item is going at end of list;
        case repo.one(list_count_query) do # get count of list items
          count when new_idx < count -> {:ok, new_idx} # use new_index if not at end of list
          count -> {:ok, count - 1} # set index to count-1 if at end of list (0 based index)
        end
    end)
    |> multi_update_all({:dec_positions, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
      where: field(t, ^list_key_field) == ^list_key_value, # get all items in list
      where: field(t, ^struct_key_field) != ^struct_key_value, # except struct being moved
      # and where item.position > old position and <= new position
      where: t.position > subquery(old_position_query) and t.position <= ^computed_index,
      update: [inc: [position: -1]] # decrement positions because shifting these up the list
      )
    end)
    |> multi_update_all({:inc_positions, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
      where: field(t, ^list_key_field) == ^list_key_value, # get all items in list
      where: field(t, ^struct_key_field) != ^struct_key_value, # except struct being moved
      # and where item.position < old position and >= new position
      where: t.position < subquery(old_position_query) and t.position >= ^computed_index,
      update: [inc: [position: 1]] # increment positions because shifting these down the list
      )
    end)
    |> multi_update_all({:position, name}, fn %{{:index, ^name} => computed_index} ->
      from(t in type,
        where: field(t, ^struct_key_field) == ^struct_key_value, # get item being moved
        update: [set: [position: ^computed_index]] # assign it new position
      )
    end)
  end