FTR: I came up with this solution to mark the seeds as already applied from within seeds.exs, so that running that script is idempotent.

defmodule SeedsIdempotencyHandling do
  @magic_marker_value 2000_00_00_00_00_00
  import Ecto.Query, only: [from: 2]

  def seeds_already_applied? do
    query = from m in Ecto.Migration.SchemaMigration, where: m.version == @magic_marker_value
    Repo.exists?(query)
  end

  def mark_seeds_as_applied do
    Repo.insert(%Ecto.Migration.SchemaMigration{
      version: @magic_marker_value,
      inserted_at: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
    })
  end
end

if SeedsIdempotencyHandling.seeds_already_applied?() do
  IO.puts("Seeds already applied, skipping...")
else
  # put your seeds here
  SeedsIdempotencyHandling.mark_seeds_as_applied()
end