Ecto.Type: how to `cast` into a changeset like `embeds_{one,many}`?

This is a follow-on to Clearing an `embeds_many` entry (`cast_embed/3`), where I have a JSONB column with a constraint of NULL or non-empty list; embeds_many is inappropriate because it treats [] as the nil value for the field, and we explicitly do not want that behaviour. I consider it to be fundamentally incorrect, even though I understand it was done because it “normalizes” embeds as if they were associations.

I have created an Ecto.ParameterizedType for an OptionalList, derived at least partially from @mathieuprog’s polymorphic_embed library and some other digging around that I did.

When used with an embedded schema, the biggest problem (which is also true of polymorphic_embed) is that, unlike cast_embed, the casting of the resulting object is not an %Ecto.Changeset{} for the objects in the list, but is a realized version of the struct and it is doing no changeset validation.

I cannot figure out how to make this work, because when the field is loaded from the database, if it is not NULL, I want it to to cast into [%Value{}, %Value{}, …], but when taking data in for a changeset, I want it to cast into [%Ecto.Changeset{}, %Ecto.Changeset{}, …].

It appears that Ecto.Embedded has a prepare function — but I can’t quite figure out how it gets called in the process, whether it is something that can be done for automatically with a regular Ecto.Type, or whether I have to implement functions similar to cast_embed.

As I mentioned, I am seeing similar behaviour with polymorphic_embed, and I want to fix it in my fork so that it might be able to be upstreamed once I’m sure it works.

defmodule EctoOptionalList do
  @moduledoc """
  A nullable list of the provided type.

  This type wraps `{:array, :map}` in the same way that `embed_many/3` does,
  except that `nil` is a way of clearing the list completely, whereas
  `embed_many/3` always stores a list.

  ### Example

      field :optional, EctoOptionalList, type: Test.Option
  """

  use Ecto.ParameterizedType

  alias Ecto.Schema.Loader

  defstruct [:type, :field, :schema, :load_type]

  @impl true
  def type(_params), do: {:array, :map}

  @impl true
  def init(opts) do
    type = validate_type(opts[:type])

    load_type =
      if function_exported?(type, :__schema__, 1) do
        :schema
      else
        :struct
      end

    struct(__MODULE__, Keyword.merge(opts, type: type, load_type: load_type))
  end

  @impl true
  def cast(nil, _params), do: {:ok, nil}
  def cast(value, _params) when not is_list(value), do: :error

  def cast(list, %{type: type}) do
    result =
      Enum.reduce_while(list, [], fn
        %^type{} = entry, acc -> {:cont, [entry | acc]}
        %_badtype{}, _acc -> {:halt, :error}
        %{} = entry, acc -> {:cont, [build!(type, entry) | acc]}
      end)

    case result do
      :error -> :error
      value -> {:ok, value}
    end
  end

  @impl true
  def load(nil, _, _), do: {:ok, nil}
  def load(data, _, _) when not is_list(data), do: :error

  def load(data, fun, %{type: type, load_type: load_type, field: field}) do
    {:ok, Enum.map(data, &load_field(load_type, type, field, &1, fun))}
  end

  @impl true
  def dump(nil, _, _), do: {:ok, nil}
  def dump(data, _, _) when not is_list(data), do: :error

  def dump(data, fun, %{type: type, load_type: load_type, field: field}) do
    types = if load_type == :schema, do: type.__schema__(:dump)

    {:ok, Enum.map(data, &dump_field(load_type, type, field, &1, types, fun))}
  end

  defp validate_type(nil), do: invalid_type!()
  defp validate_type(%mod{}), do: validate_type(mod)
  defp validate_type(value) when not is_atom(value), do: invalid_type!()

  defp validate_type(mod) do
    with {:module, _} <- Code.ensure_compiled(mod),
         true <- function_exported?(mod, :__struct__, 0) do
      mod
    else
      _ -> invalid_type!()
    end
  end

  defp invalid_type! do
    raise ArgumentError, """
    EctoOptionalList must have a `type` option referencing a struct module.
    This option was either omitted or the value provided is neither a struct
    nor a module defining a struct.

    For example:

        field :my_field, EctoOptionalList, type: MyFieldStruct
        field :my_field, EctoOptionalList, type: %MyFieldStruct{}
    """
  end

  defp build!(type, value) when is_map(value) do
    data =
      Enum.map(
        value,
        fn
          {k, v} when is_binary(k) -> {String.to_existing_atom(k), v}
          kv -> kv
        end
      )

    struct!(type, data)
  end

  defp load_field(:struct, type, _field, value, _loader) when is_map(value) do
    build!(type, value)
  end

  defp load_field(:schema, type, _field, value, loader) when is_map(value) do
    Loader.unsafe_load(type, value, loader)
  end

  defp load_field(_loader_type, type, field, value, _loader) do
    raise ArgumentError,
          "cannot load `#{field}`, expected a map for #{type} but got: #{inspect(value)}"
  end

  defp dump_field(:struct, type, _field, %{__struct: type} = value, _types, _dumper) do
    Map.from_struct(value)
  end

  defp dump_field(:schema, type, _field, %{__struct__: type} = value, types, dumper) do
    Loader.safe_dump(value, types, dumper)
  end

  defp dump_field(_loader_type, type, field, value, _types, _dumper) do
    raise ArgumentError,
          "cannot dump `#{field}`, expected a list of #{type} struct values but got: #{inspect(value)}"
  end
end