Migrations not being run/detected on a deployed mix release

I’m running a Phoenix app on a Kubernetes cluster (through Digital Ocean) and I was able to deploy my app with minimal issues for the most part.

However, for some reason, calling Ecto.Migrator keeps saying “migrations already up” when in reality, it hasn’t actually done anything. Even the schema_migrations table is empty.

This isn’t the first time I’ve done a deployment setup like this so I am dumbfounded as to why this is happening in the first place. I definitely have a few migration files in the app, but they have not run at all.

Here is the release task module that I’m using:

defmodule MyApp.ReleaseTasks do
  @moduledoc false
  @app :my_app

  def migrate do
    load_app()

    # migrate database
    for repo <- repos() do
      {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true))
    end

    # migrate event store
    config = MyApp.EventStore.config()
    :ok = EventStore.Tasks.Migrate.exec(config, [])
  end

  def rollback(repo, version) do
    load_app()
    {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
  end

  def seed do
    for repo <- repos() do
      {:ok, _, _} = Ecto.Migrator.with_repo(repo, &MyApp.ReleaseTasks.run_seed_for(&1))
    end
  end

  def init_event_store do
    load_app()

    config = MyApp.EventStore.config()
    :ok = EventStore.Tasks.Init.exec(config, [])
  end

  def run_seed_for(repo) do
    seed_script = priv_path_for(repo, "seeds.exs")

    if File.exists?(seed_script) do
      IO.puts("Running seed script...")
      Code.eval_file(seed_script)
    end
  end

  defp repos do
    Application.fetch_env!(@app, :ecto_repos)
  end

  defp load_app do
    {:ok, _} = Application.ensure_all_started(:postgrex)
    {:ok, _} = Application.ensure_all_started(:ssl)

    :ok = Application.load(@app)
  end

  defp priv_path_for(repo, filename) do
    app = Keyword.get(repo.config, :otp_app)

    repo_underscore =
      repo
      |> Module.split()
      |> List.last()
      |> Macro.underscore()

    priv_dir = "#{:code.priv_dir(app)}"

    Path.join([priv_dir, repo_underscore, filename])
  end
end

This is my app’s config/releases.ex file:

import Config

config :my_app, MyAppWeb.Endpoint,
  http: [
    port: String.to_integer(System.fetch_env!("PORT")),
    transport_options: [socket_opts: [:inet6]]
  ],
  url: [
    host: System.fetch_env!("HOST"),
    port: String.to_integer(System.fetch_env!("PORT"))
  ],
  server: true,
  secret_key_base: System.fetch_env!("SECRET_KEY_BASE")

config :my_app, MyApp.Repo,
  hostname: System.fetch_env!("DB_HOSTNAME"),
  username: System.fetch_env!("DB_USERNAME"),
  password: System.fetch_env!("DB_PASSWORD"),
  database: System.fetch_env!("DB_NAME"),
  port: System.fetch_env!("DB_PORT"),
  pool_size: String.to_integer(System.fetch_env!("DB_POOL_SIZE")),
  ssl: true,
  ssl_opts: [
    cacertfile: System.fetch_env!("DB_CERT")
  ]

config :my_app, MyApp.EventStore,
  hostname: System.fetch_env!("EVENTSTORE_DB_HOSTNAME"),
  username: System.fetch_env!("EVENTSTORE_DB_USERNAME"),
  password: System.fetch_env!("EVENTSTORE_DB_PASSWORD"),
  database: System.fetch_env!("EVENTSTORE_DB_NAME"),
  port: System.fetch_env!("EVENTSTORE_DB_PORT"),
  pool_size: String.to_integer(System.fetch_env!("EVENTSTORE_DB_POOL_SIZE")),
  ssl: true,
  ssl_opts: [
    cacertfile: System.fetch_env!("EVENTSTORE_DB_CERT")
  ]

This is the Dockerfile that I’m using:

FROM beardedeagle/alpine-phoenix-builder:1.11.3 AS build
WORKDIR /app
RUN mix local.hex --force && \
    mix local.rebar --force
ENV MIX_ENV=prod
COPY mix.exs mix.lock ./
COPY config config
RUN mix do deps.get, deps.compile
COPY lib lib
COPY rel rel
RUN mix do compile, release

FROM alpine:3.13 AS app
RUN apk add --no-cache openssl ncurses-libs
WORKDIR /app
EXPOSE 4000
RUN chown nobody:nobody /app
USER nobody:nobody
COPY --from=build --chown=nobody:nobody /app/_build/prod/rel/my_app ./
ENV HOME=/app
CMD ["bin/my_app", "start"]

I can confirm that the required environment variables are present inside the pod that gets created from the Kubernetes deployment so I don’t think there are any issues with that.

Also, running the commands from the release tasks module don’t result in any errors. The only issue is that running bin/my_app eval "MyApp.ReleaseTasks.migrate outputs “Migrations already up” when that is not in fact the case.

Another thing I could point out is that I am using commanded/eventstore for this app and running the migrations for the event store actually works.

Any help is appreciated.

1 Like

I deploy with docker, but I don’t use Kubernetes, instead plain docker-compose.yml file.

I don’t see how I can help you with your issue, but you can try out my docker-compose.yml file:

version: "2.3"

services:
  release:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    build:
      context: .
      args:
        ELIXIR_VERSION: ${ELIXIR_VERSION? Missing env var for ELIXIR_VERSION}
        OTP_VERSION: ${ERLANG_OTP_VERSION? Missing env var for ERLANG_OTP_VERSION}
        ALPINE_VERSION: ${ALPINE_VERSION? Missing env var for ALPINE_VERSION}
        MIX_ENV: "prod"

  migrate:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    env_file:
      - .env
    command:
      - eval
      - Rumbl.ReleaseTasks.migrate
    healthcheck:
      test: "exit 0"
    depends_on:
      database:
        condition: service_healthy
    networks:
      - default

  app:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    restart: unless-stopped
    env_file:
      - .env
    # Uncomment if not behind Traefik
    #ports:
      #- 0.0.0.0:${PUBLIC_DOMAIN_PORT}:${SERVER_HTTP_PORT}
    networks:
      - traefik
      - default
    depends_on:
      migrate:
        condition: service_started
      database:
        condition: service_healthy
    labels:
      - "traefik.enable=true"
      - "traefik.backend=${PUBLIC_DOMAIN? Missing env var PUBLIC_DOMAIN}"
      - "traefik.docker.network=traefik"
      - "traefik.port=${SERVER_HTTP_PORT}"
      - "traefik.frontend.rule=Host:${PUBLIC_DOMAIN}"

  database:
    image: postgres:10-alpine
    restart: unless-stopped
    healthcheck:
      test: "exit 0"
    env_file:
      - .env
    environment:
      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - ${DATABASE_DIR? Missing env var DATABASE_DIR}/.${APP_NAME}/database:/var/lib/postgresql/data
    networks:
      - default

networks:
  traefik:
    external: true

If you continue to see the problem with migrations with this docker-compose file then you must have some issue in the Kubernetes configuration.

The .env file to be at same level of docker-compose.yml:

# .env

# Adjust as you see fit
DOCKER_IMAGE=public_domain:tag

ELIXIR_VERSION=1.11.3
ERLANG_OTP_VERSION=23.2.2
ALPINE_VERSION=3.12.1

SERVER_HOSTNAME=localhost
SERVER_HTTP_PORT=4000

PUBLIC_SCHEME=https
PUBLIC_DOMAIN=example.com
PUBLIC_DOMAIN_PORT=443

Build the release with:

docker-compose build release

Run the release with:

docker-compose up --detach app

I also use a Docker image based on the official guide, but I have fixed the insecure use of nobody user:

My Dockerfile:

ARG ELIXIR_VERSION
ARG OTP_VERSION
ARG ALPINE_VERSION

FROM hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-alpine-${ALPINE_VERSION} as build

ARG MIX_ENV=prod
ARG BUILD_RELEASE_FROM=master

ENV MIX_ENV=${MIX_ENV}

WORKDIR /app

RUN \
  apk add \
    --no-cache \
    openssh-client \
    build-base \
    npm \
    git \
    python3 && \

  mix local.hex --force && \
  mix local.rebar --force

COPY .env /release/.env
COPY ./.git /workspace

RUN \
  git clone --local /workspace . && \
  git checkout "${BUILD_RELEASE_FROM}" && \

  mix deps.get --only "${MIX_ENV}" && \

  npm --prefix ./assets ci --progress=false --no-audit --loglevel=error && \
  npm run --prefix ./assets deploy && \
  mix phx.digest && \

  mix compile && \
  mix release

# Start a new build stage so that the final image will only contain
# the compiled release and other runtime necessities
FROM alpine:${ALPINE_VERSION} AS app

ENV USER="phoenix"
ENV HOME=/home/"${USER}"
ENV APP_DIR="${HOME}/app"

RUN \
  apk upgrade --no-cache && \
  apk add --no-cache \
    openssl \
    ncurses-libs && \

  # Creates a unprivileged user to run the app
  addgroup \
   -g 1000 \
   -S "${USER}" && \
  adduser \
   -s /bin/sh \
   -u 1000 \
   -G "${USER}" \
   -h "${HOME}" \
   -D "${USER}" && \

  su "${USER}" sh -c "mkdir ${APP_DIR}"

# Everything from this line onwards will run in the context of the unprivileged user.
USER "${USER}"

WORKDIR "${APP_DIR}"

COPY --from=build --chown="${USER}":"${USER}" /app/_build/prod/rel/tasks ./

ENTRYPOINT ["./bin/tasks"]

# Docker Usage:
#  * build: sudo docker build -t phoenix/tasks .
#  * shell: sudo docker run --rm -it --entrypoint "" -p 80:4000 -p 443:4040 phoenix/tasks sh
#  * run:   sudo docker run --rm -it -p 80:4000 -p 443:4040 --env-file .env --name tasks phoenix/tasks
#  * exec:  sudo docker exec -it tasks sh
#  * logs:  sudo docker logs --follow --tail 10 tasks
#
# Extract the production release to your host machine with:
#
# ```
# sudo docker run --rm -it --entrypoint "" --user $(id -u) -v "$PWD/_build:/home/phoenix/_build"  phoenix/tasks sh -c "tar zcf /home/phoenix/_build/app.tar.gz ."
# ls -al _build
# ````
CMD ["start"]

3 Likes

Can you dump the contents of the migrations table and make sure somehow those rows didn’t get added?

1 Like

Your suggestion made me look back at my Dockerfile line by line and I found out that I wasn’t actually copying the priv folder.

Rookie mistake, but it’s fixed now. I even removed the use of nobody as mentioned in the link that you shared.

Here is the final Dockerfile that I’m using now:

FROM hexpm/elixir:1.11.4-erlang-23.3.1-alpine-3.13.3 AS build
RUN apk add --no-cache build-base npm git python3
WORKDIR /app
RUN mix local.hex --force && \
    mix local.rebar --force
ENV MIX_ENV=prod
COPY mix.exs mix.lock ./
RUN mix deps.get --only $MIX_ENV
RUN mkdir config
COPY config/config.exs config/releases.exs config/$MIX_ENV.exs config/
RUN mix deps.compile
COPY priv priv
COPY lib lib
COPY rel rel
RUN mix do compile, release

FROM alpine:3.13 AS app
RUN apk add --no-cache openssl ncurses-libs
ENV USER="phoenix"
ENV HOME=/home/"${USER}"
ENV APP_DIR="${HOME}/app"
RUN \
  addgroup \
   -g 1000 \
   -S "${USER}" && \
  adduser \
   -s /bin/sh \
   -u 1000 \
   -G "${USER}" \
   -h "${HOME}" \
   -D "${USER}" && \
  su "${USER}" sh -c "mkdir ${APP_DIR}"
USER "${USER}"
WORKDIR "${APP_DIR}"
EXPOSE 4000
COPY --from=build --chown="${USER}":"${USER}" /app/_build/prod/rel/my_app ./
ENTRYPOINT ["bin/my_app"]
CMD ["start"]
2 Likes