How to do migrations on Elixir 1.9's mix release?

Questions / Help
1

With the release of Elixir 1.9, we have the option to not use Distillery for creating app releases anymore.

While I understand that we can still use the ReleaseTasks from Distillery, I still can’t get it to work because it doesn’t see the database at runtime.

# config/config.exs

import Config

config :my_app,
  ecto_repos: [MyApp.Repo]

# Configures the endpoint
config :my_app, MyAppWeb.Endpoint,
  url: [host: "localhost"],
  secret_key_base: "SyIMuifvLpc1Wtd0XQWhY9ajIKfwRdExf5L0yKrXM33o9yLEPAJ94L4UjqMRehLc",
  render_errors: [view: MyAppWeb.ErrorView, accepts: ~w(json)],
  pubsub: [name: MyApp.PubSub, adapter: Phoenix.PubSub.PG2]

# Configures Elixir's Logger
config :logger, :console,
  format: "$time $metadata[$level] $message\n",
  metadata: [:request_id]

# Use Jason for JSON parsing in Phoenix
config :phoenix, :json_library, Jason

import_config "#{Mix.env()}.exs"

# config/prod.exs

import Config

config :my_app, MyAppWeb.Endpoint,
  check_origin: false

config :logger, level: :info

# config/releases.exs

import Config

config :my_app, MyAppWeb.Endpoint,
  http: [port: String.to_integer(System.fetch_env!("PORT"))],
  url: [host: System.fetch_env!("HOST"), port: String.to_integer(System.fetch_env!("PORT"))],
  server: true,
  secret_key_base: System.fetch_env!("SECRET_KEY_BASE")

config :my_app, MyApp.Repo,
  hostname: System.fetch_env!("DB_HOSTNAME"),
  username: System.fetch_env!("DB_USERNAME"),
  password: System.fetch_env!("DB_PASSWORD"),
  database: System.fetch_env!("DB_NAME"),
  pool_size: String.to_integer(System.fetch_env!("DB_POOL_SIZE"))

# lib/release_tasks.ex

defmodule MyApp.ReleaseTasks do
  @start_apps [
    :crypto,
    :ssl,
    :postgrex,
    :ecto,
    :ecto_sql # If using Ecto 3.0 or higher
  ]

  @repos Application.get_env(:my_app, :ecto_repos, [])

  def migrate() do
    start_services()

    run_migrations()

    stop_services()
  end

  def seed() do
    start_services()

    run_migrations()

    run_seeds()

    stop_services()
  end

  defp start_services do
    IO.puts("Starting dependencies..")
    # Start apps necessary for executing migrations
    Enum.each(@start_apps, &Application.ensure_all_started/1)

    # Start the Repo(s) for app
    IO.puts("Starting repos..")

    # pool_size can be 1 for ecto < 3.0
    Enum.each(@repos, & &1.start_link(pool_size: 2))
  end

  defp stop_services do
    IO.puts("Success!")
    :init.stop()
  end

  defp run_migrations do
    Enum.each(@repos, &run_migrations_for/1)
  end

  defp run_migrations_for(repo) do
    app = Keyword.get(repo.config(), :otp_app)
    IO.puts("Running migrations for #{app}")
    migrations_path = priv_path_for(repo, "migrations")
    Ecto.Migrator.run(repo, migrations_path, :up, all: true)
  end

  defp run_seeds do
    Enum.each(@repos, &run_seeds_for/1)
  end

  defp run_seeds_for(repo) do
    # Run the seed script if it exists
    seed_script = priv_path_for(repo, "seeds.exs")

    if File.exists?(seed_script) do
      IO.puts("Running seed script..")
      Code.eval_file(seed_script)
    end
  end

  defp priv_path_for(repo, filename) do
    app = Keyword.get(repo.config(), :otp_app)

    repo_underscore =
      repo
      |> Module.split()
      |> List.last()
      |> Macro.underscore()

    priv_dir = "#{:code.priv_dir(app)}"

    Path.join([priv_dir, repo_underscore, filename])
  end
end

$ MIX_ENV=prod mix release
$ PORT="4000" HOST="127.0.0.1" SECRET_KEY_BASE="JdgV1bdEtGnR2hM4DFXt9+gJnLfSSOssp1Z8YEpnNbXiBhvYYZ14WwYRMlDGGhLQ" DB_HOSTNAME=localhost DB_USERNAME=postgres DB_PASSWORD=postgres DB_NAME=my_app_dev DB_POOL_SIZE="15" _build/prod/rel/my_app/bin/my_app eval "MyApp.ReleaseTasks.migrate"
Starting dependencies..
Starting repos..
Running migrations for my_app
14:31:03.619 [error] GenServer #PID<0.217.0> terminating
** (RuntimeError) connect raised KeyError exception: key :database not found. The exception details are hidden, as they may contain sensitive data such as database credentials. You may set :show_sensitive_data_on_connection_error to true when starting your connection if you wish to see all of the details
    (elixir) lib/keyword.ex:393: Keyword.fetch!/2
    (postgrex) lib/postgrex/protocol.ex:90: Postgrex.Protocol.connect/1
    (db_connection) lib/db_connection/connection.ex:69: DBConnection.Connection.connect/2
    (connection) lib/connection.ex:622: Connection.enter_connect/5
    (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: nil
14:31:03.619 [error] GenServer #PID<0.218.0> terminating
** (RuntimeError) connect raised KeyError exception: key :database not found. The exception details are hidden, as they may contain sensitive data such as database credentials. You may set :show_sensitive_data_on_connection_error to true when starting your connection if you wish to see all of the details
    (elixir) lib/keyword.ex:393: Keyword.fetch!/2
    (postgrex) lib/postgrex/protocol.ex:90: Postgrex.Protocol.connect/1
    (db_connection) lib/db_connection/connection.ex:69: DBConnection.Connection.connect/2
    (connection) lib/connection.ex:622: Connection.enter_connect/5
    (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: nil
14:31:03.622 [error] GenServer #PID<0.220.0> terminating
** (RuntimeError) connect raised KeyError exception: key :database not found. The exception details are hidden, as they may contain sensitive data such as database credentials. You may set :show_sensitive_data_on_connection_error to true when starting your connection if you wish to see all of the details
    (elixir) lib/keyword.ex:393: Keyword.fetch!/2
    (postgrex) lib/postgrex/protocol.ex:90: Postgrex.Protocol.connect/1
    (db_connection) lib/db_connection/connection.ex:69: DBConnection.Connection.connect/2
    (connection) lib/connection.ex:622: Connection.enter_connect/5
    (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: nil
14:31:03.622 [error] GenServer #PID<0.219.0> terminating
** (RuntimeError) connect raised KeyError exception: key :database not found. The exception details are hidden, as they may contain sensitive data such as database credentials. You may set :show_sensitive_data_on_connection_error to true when starting your connection if you wish to see all of the details
    (elixir) lib/keyword.ex:393: Keyword.fetch!/2
    (postgrex) lib/postgrex/protocol.ex:90: Postgrex.Protocol.connect/1
    (db_connection) lib/db_connection/connection.ex:69: DBConnection.Connection.connect/2
    (connection) lib/connection.ex:622: Connection.enter_connect/5
    (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: nil
** (EXIT from #PID<0.164.0>) shutdown

In Distillery, I would use "${DB_NAME}" instead of System.fetch_env!("DB_NAME") and setting REPLACE_OS_VARS=true and the above code/commands will work. It doesn’t work for Elixir 1.9’s release command though.

Do I need to define the database config somewhere else?

1 Like
2

Upgrade to latest Ecto SQL and do this:

For environment variables, you can rename config/prod.secret.exs to config/releases.exs and regularly access System.fetch_env!(“DB_NAME”) in there. See the release docs for more info.

14 Likes
3

That fixed it! Thanks!

2 Likes
4

Another big plus to using config/releases.exs is that you can use your application code.

I use this in order to parse JSON config objects using Jason.

2 Likes
6

Google brought me here today, but it turns out, there exists a much simpler solution now: https://hexdocs.pm/phoenix/releases.html#ecto-migrations-and-custom-commands - much less custom code to add to release tasks.

6 Likes
7

@josevalim how can I create the database on a release task?

8

Something like this, maybe:

def createdb do
    Enum.each(repos(), fn repo ->
      repo.__adapter__.storage_up(repo.config)
    end)
end
3 Likes