Clearing an `embeds_many` entry (`cast_embed/3`)

I seem to remember people around here saying that Ecto.Changeset is not treating nil as an actual change but don’t quote me on that.

But in case it’s true, what’s the difficulty in making the empty list the special empty value you’re looking for?

Because it doesn’t seem to work. I have added the following to my .iex.exs for my project to explore.

defmodule Test.Flow do
  use Ecto.Schema
  import Ecto.Changeset

  defmodule Option do
    use Ecto.Schema
    import Ecto.Changeset

    @primary_key false
    embedded_schema do
      field :value, :string
    end

    def changeset(opts \\ %__MODULE__{}, attrs) do
      cast(opts, attrs, [:value])
    end
  end

  schema "flows" do
    embeds_many :options, Option, on_replace: :delete
  end

  def changeset(flows \\ %__MODULE__{}, attrs) do
    flows
    |> cast(attrs, [:id])
    |> cast_embed(:options)
    |> tap(fn changeset ->
      IO.inspect(fetch_change(changeset, :options))
    end)
    |> dbg()
  end
end

And this is my experimental session (I just dropped and recreated the table in psql):

iex(1)> Repo.insert!(Test.Flow.changeset(%{options: [%{}]}))
{:ok,
 [
   #Ecto.Changeset<action: :insert, changes: %{}, errors: [],
    data: #Test.Flow.Option<>, valid?: true>
 ]}
[.iex.exs:156: Test.Flow.changeset/2]
flows #=> %Test.Flow{
  __meta__: #Ecto.Schema.Metadata<:built, "flows">,
  id: nil,
  options: []
}
|> cast(attrs, [:id]) #=> #Ecto.Changeset<action: nil, changes: %{}, errors: [], data: #Test.Flow<>,
 valid?: true>
|> cast_embed(:options) #=> #Ecto.Changeset<
  action: nil,
  changes: %{
    options: [
      #Ecto.Changeset<action: :insert, changes: %{}, errors: [],
       data: #Test.Flow.Option<>, valid?: true>
    ]
  },
  errors: [],
  data: #Test.Flow<>,
  valid?: true
>
|> tap(fn changeset -> IO.inspect(fetch_change(changeset, :options)) end) #=> #Ecto.Changeset<
  action: nil,
  changes: %{
    options: [
      #Ecto.Changeset<action: :insert, changes: %{}, errors: [],
       data: #Test.Flow.Option<>, valid?: true>
    ]
  },
  errors: [],
  data: #Test.Flow<>,
  valid?: true
>

%Test.Flow{
  __meta__: #Ecto.Schema.Metadata<:loaded, "flows">,
  id: 1,
  options: [%Test.Flow.Option{value: nil}]
}
iex(2)> Test.Flow.changeset(Repo.get(Test.Flow, 1), %{options: []})
iex(2)> Test.Flow.changeset(Repo.get(Test.Flow, 1), %{options: []})
{:ok,
 [
   #Ecto.Changeset<action: :replace, changes: %{}, errors: [],
    data: #Test.Flow.Option<>, valid?: true>
 ]}
[.iex.exs:156: Test.Flow.changeset/2]
flows #=> %Test.Flow{
  __meta__: #Ecto.Schema.Metadata<:loaded, "flows">,
  id: 1,
  options: [%Test.Flow.Option{value: nil}]
}
|> cast(attrs, [:id]) #=> #Ecto.Changeset<action: nil, changes: %{}, errors: [], data: #Test.Flow<>,
 valid?: true>
|> cast_embed(:options) #=> #Ecto.Changeset<
  action: nil,
  changes: %{
    options: [
      #Ecto.Changeset<action: :replace, changes: %{}, errors: [],
       data: #Test.Flow.Option<>, valid?: true>
    ]
  },
  errors: [],
  data: #Test.Flow<>,
  valid?: true
>
|> tap(fn changeset -> IO.inspect(fetch_change(changeset, :options)) end) #=> #Ecto.Changeset<
  action: nil,
  changes: %{
    options: [
      #Ecto.Changeset<action: :replace, changes: %{}, errors: [],
       data: #Test.Flow.Option<>, valid?: true>
    ]
  },
  errors: [],
  data: #Test.Flow<>,
  valid?: true
>

#Ecto.Changeset<
  action: nil,
  changes: %{
    options: [
      #Ecto.Changeset<action: :replace, changes: %{}, errors: [],
       data: #Test.Flow.Option<>, valid?: true>
    ]
  },
  errors: [],
  data: #Test.Flow<>,
  valid?: true
>
iex(3)> Repo.update!(v())
{:ok,
 %Test.Flow{
   __meta__: #Ecto.Schema.Metadata<:loaded, "flows">,
   id: 1,
   options: []
 }}

But that is incorrect. If the array is empty, the field should be set to NULL, and that’s clearly not the case:

psql> select * from flows;
┌────┬─────────┐
│ id │ options │
├────┼─────────┤
│  1 │ []      │
└────┴─────────┘
(1 row)

I can change the wire serialization so that an empty array is returned as null, but it feels odd that a nullable field can no longer be set to null — and I can’t see any way from the fetch_change to know that this is going to be cleared. I would have to refer back to attrs to see if I should do anything.

This should be:

Repo.insert!(Test.Flow.changeset(%{options: []}))

IE, don’t include the empty map as that implies you have an option you want to run through a changeset. You just want an empty list meaning “no options.” Otherwise ya, AFAIK, Ecto’s has_many relations are always going to be lists (ie, not null). There was some discussion around this that I can’t find atm.

That initial insert was to provide a record that needed to have [{value: null}] turned into null through clearing to illustrate my problem. The later changeset calls illustrate this more clearly.

My issue is with embeds_many, not has_many. If I were storing this data in a separate table, this would not be a problem, because the “owned” records would simply be deleted.

This feels like something that can and should be handled better. Maybe with something like embeds_many :field, Type, nullable: true, which would theoretically be able to flag to cast_embed/4 that if the value is nil (and not missing), then it should be set to nil in the update.

I’m going to need to figure a way around this, because an empty list is explicitly not a valid value. It’s either nil or a non-empty list.

Sorry, I meant embeds_many—they work the same in that regard. And also I seem to have misread your iex output.

I found the discussion I was thinking of:

Thanks for the link.

Yeah. I have explicitly disallowed the empty list in the type definition, which is why this behaviour is…disappointing. I may need to see if there is a way to make embeds_many and cast_embed work properly for this case, because as a few posts in that thread make clear, NULL is not the same as [] and we’re just somewhat lucky that I hadn’t made a constraint on the table which checked for null or jsonb_array_length(options) > 1 (this is still a good idea, but until I can work around this issue, I can’t have it).

You could add your voice to that thread. Not sure if it will do much good but you can always try! My guess is that this just doesn’t come up often enough and there are other ways around it, though I’m not quite sure of the workarounds myself.

One workaround is to define an Ecto.Type. My example here isn’t well organized (the type here should be a different module, OptionList), but that’s fine for an example:

defmodule Test.Flow do
  use Ecto.Schema
  import Ecto.Changeset

  defmodule Option do
    use Ecto.Schema
    use Ecto.Type

    import Ecto.Changeset

    @primary_key false
    embedded_schema do
      field :value, :string
    end

    def changeset(opts \\ %__MODULE__{}, attrs) do
      cast(opts, attrs, [:value])
    end

    @impl Ecto.Type
    def type, do: {:array, :map}

    @impl Ecto.Type
    def cast([_ | _] = list) do
      cond do
        Enum.all?(list, &is_struct(&1, __MODULE__)) -> {:ok, list}
        Enum.all?(list, &is_map/1) -> load(list)
        true -> :error
      end
    end

    def cast(nil), do: {:ok, nil}
    def cast(_), do: :error

    @impl Ecto.Type
    def load(nil), do: {:ok, nil}

    def load(data) when is_list(data) do
      {
        :ok,
        Enum.map(data, fn entry ->
          struct!(
            __MODULE__,
            for {k, v} <- entry do
              {String.to_existing_atom(k), v}
            end
          )
        end)
      }
    end

    @impl Ecto.Type
    def dump([_ | _] = list) do
      if Enum.all?(list, &match?(%__MODULE__{}, &1)) do
        {:ok, Enum.map(list, &Map.from_struct/1)}
      else
        :error
      end
    end

    def dump(nil), do: {:ok, nil}
    def dump(_), do: :error
  end

  schema "flows" do
    field :options, Option
  end

  def changeset(flows \\ %__MODULE__{}, attrs) do
    cast(flows, attrs, [:id, :options])
  end
end

It’s a lot of work, and I suspect that some of it could be wrapped in a macro.

I fear that the lesson here is to avoid embeds_many if your column is intentionally nullable, because Ecto works against your design in this case.