Ecto Many-to-Many Association table does not append on insert?

I have Users and Orgs. Users can belong to multiple Orgs and Orgs can have many users:

defmodule Elijah.Schema.User do
  use Ecto.Schema
  import Ecto.Changeset
  use Waffle.Ecto.Schema

...
  schema "users" do
    field :email, :string
    field :username, :string
    field :name, :string
    field :phone_number, :string
    field :confirmed_at, :naive_datetime
    field :is_admin, :boolean
    field :avatar, Elijah.AvatarUploader.Type

    many_to_many :orgs, Elijah.Schema.Org, join_through: "orgs_users", on_replace: :delete

    timestamps()
  end
...

Org:

defmodule Elijah.Schema.Org do
  use Ecto.Schema
  import Ecto.Changeset
  alias Elijah.Type.OrgHashId
...
  schema "orgs" do
    field :basename, :string
    field :shortname, :string
    field :uuid, Ecto.UUID
    field :hash_id, :string

    has_many :channels, Elijah.Schema.Channel
    has_many :playlists, Elijah.Schema.Playlist
    many_to_many :users, Elijah.Schema.User, join_through: "orgs_users", on_replace: :delete

    timestamps(type: :utc_datetime)

    # timestamps()
  end
...

OrgsUsers:

defmodule Elijah.Schema.OrgsUsers do
  use Ecto.Schema
  import Ecto.Changeset


  schema "orgs_users" do
    field :org_id, :integer
    field :user_id, :integer
  end

  @doc false
  def changeset(orgs_users, attrs) do
    orgs_users
    |> cast(attrs, [:user_id, :org_id])
    |> validate_required([:user_id, :resource_id])
  end
end

I made a function called Accounts.add_user() which is meant to add a user to an org and update the orgs_users table. However what is happening is that the second association will clobber the first one:

Accounts.add_user():

    def add_user(%{"org_id" => org_id, "user_id" => user_id}) do
      org = Repo.get_by!(Org, id: org_id)
      user = Repo.get_by!(User, id: user_id)
      org
      |> Repo.preload(:users)
      |> Ecto.Changeset.change()
      |> Ecto.Changeset.put_assoc(:users, [user])
      |> Repo.update!
    end

Here is the log where I add two users:

SELECT * from orgs_users:

elijah_dev=# select * from orgs_users;
 id | org_id | user_id 
----+--------+---------
  2 |      1 |       2
(1 row)

Any idea why add_user() does not append? I also tried:

Ecto.Changeset.put_assoc(:users, org.users ++ user)

but resulted in this error:

** (ArgumentError) argument error
    :erlang.++(#Ecto.Association.NotLoaded<association :users is not loaded>, #Elijah.Schema.User<__meta__: #Ecto.Schema.Metadata<:loaded, "users">, avatar: nil, confirmed_at: nil, email: "amos@elijah.app", id: 1, inserted_at: ~N[2020-10-19 21:00:31], is_admin: nil, name: "Amos", orgs: #Ecto.Association.NotLoaded<association :orgs is not loaded>, phone_number: nil, updated_at: ~N[2020-10-19 21:00:31], username: "amos", ...>)
    (faithful_word 0.1.0) lib/faithful_word/accounts.ex:93: Elijah.Accounts.add_user/1

(first insert)