Migrations not being run/detected on a deployed mix release

terenceponce

I’m running a Phoenix app on a Kubernetes cluster (through Digital Ocean) and I was able to deploy my app with minimal issues for the most part.

However, for some reason, calling Ecto.Migrator keeps saying “migrations already up” when in reality, it hasn’t actually done anything. Even the schema_migrations table is empty.

This isn’t the first time I’ve done a deployment setup like this so I am dumbfounded as to why this is happening in the first place. I definitely have a few migration files in the app, but they have not run at all.

Here is the release task module that I’m using:

defmodule MyApp.ReleaseTasks do
  @moduledoc false
  @app :my_app

  def migrate do
    load_app()

    # migrate database
    for repo <- repos() do
      {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true))
    end

    # migrate event store
    config = MyApp.EventStore.config()
    :ok = EventStore.Tasks.Migrate.exec(config, [])
  end

  def rollback(repo, version) do
    load_app()
    {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
  end

  def seed do
    for repo <- repos() do
      {:ok, _, _} = Ecto.Migrator.with_repo(repo, &MyApp.ReleaseTasks.run_seed_for(&1))
    end
  end

  def init_event_store do
    load_app()

    config = MyApp.EventStore.config()
    :ok = EventStore.Tasks.Init.exec(config, [])
  end

  def run_seed_for(repo) do
    seed_script = priv_path_for(repo, "seeds.exs")

    if File.exists?(seed_script) do
      IO.puts("Running seed script...")
      Code.eval_file(seed_script)
    end
  end

  defp repos do
    Application.fetch_env!(@app, :ecto_repos)
  end

  defp load_app do
    {:ok, _} = Application.ensure_all_started(:postgrex)
    {:ok, _} = Application.ensure_all_started(:ssl)

    :ok = Application.load(@app)
  end

  defp priv_path_for(repo, filename) do
    app = Keyword.get(repo.config, :otp_app)

    repo_underscore =
      repo
      |> Module.split()
      |> List.last()
      |> Macro.underscore()

    priv_dir = "#{:code.priv_dir(app)}"

    Path.join([priv_dir, repo_underscore, filename])
  end
end

This is my app’s config/releases.ex file:

import Config

config :my_app, MyAppWeb.Endpoint,
  http: [
    port: String.to_integer(System.fetch_env!("PORT")),
    transport_options: [socket_opts: [:inet6]]
  ],
  url: [
    host: System.fetch_env!("HOST"),
    port: String.to_integer(System.fetch_env!("PORT"))
  ],
  server: true,
  secret_key_base: System.fetch_env!("SECRET_KEY_BASE")

config :my_app, MyApp.Repo,
  hostname: System.fetch_env!("DB_HOSTNAME"),
  username: System.fetch_env!("DB_USERNAME"),
  password: System.fetch_env!("DB_PASSWORD"),
  database: System.fetch_env!("DB_NAME"),
  port: System.fetch_env!("DB_PORT"),
  pool_size: String.to_integer(System.fetch_env!("DB_POOL_SIZE")),
  ssl: true,
  ssl_opts: [
    cacertfile: System.fetch_env!("DB_CERT")
  ]

config :my_app, MyApp.EventStore,
  hostname: System.fetch_env!("EVENTSTORE_DB_HOSTNAME"),
  username: System.fetch_env!("EVENTSTORE_DB_USERNAME"),
  password: System.fetch_env!("EVENTSTORE_DB_PASSWORD"),
  database: System.fetch_env!("EVENTSTORE_DB_NAME"),
  port: System.fetch_env!("EVENTSTORE_DB_PORT"),
  pool_size: String.to_integer(System.fetch_env!("EVENTSTORE_DB_POOL_SIZE")),
  ssl: true,
  ssl_opts: [
    cacertfile: System.fetch_env!("EVENTSTORE_DB_CERT")
  ]

This is the Dockerfile that I’m using:

FROM beardedeagle/alpine-phoenix-builder:1.11.3 AS build
WORKDIR /app
RUN mix local.hex --force && \
    mix local.rebar --force
ENV MIX_ENV=prod
COPY mix.exs mix.lock ./
COPY config config
RUN mix do deps.get, deps.compile
COPY lib lib
COPY rel rel
RUN mix do compile, release

FROM alpine:3.13 AS app
RUN apk add --no-cache openssl ncurses-libs
WORKDIR /app
EXPOSE 4000
RUN chown nobody:nobody /app
USER nobody:nobody
COPY --from=build --chown=nobody:nobody /app/_build/prod/rel/my_app ./
ENV HOME=/app
CMD ["bin/my_app", "start"]

I can confirm that the required environment variables are present inside the pod that gets created from the Kubernetes deployment so I don’t think there are any issues with that.

Also, running the commands from the release tasks module don’t result in any errors. The only issue is that running bin/my_app eval "MyApp.ReleaseTasks.migrate outputs “Migrations already up” when that is not in fact the case.

Another thing I could point out is that I am using commanded/eventstore for this app and running the migrations for the event store actually works.

Any help is appreciated.

3 comments

#ecto #docker #deployment #kubernetes

1 3658 3

2021-04-05 03:03:06 UTC

Showing Posts 1 to 3

Show Best Posts
Show All (oldest first)
Show All (newest first)

Exadra37

I deploy with docker, but I don’t use Kubernetes, instead plain docker-compose.yml file.

I don’t see how I can help you with your issue, but you can try out my docker-compose.yml file:

version: "2.3"

services:
  release:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    build:
      context: .
      args:
        ELIXIR_VERSION: ${ELIXIR_VERSION? Missing env var for ELIXIR_VERSION}
        OTP_VERSION: ${ERLANG_OTP_VERSION? Missing env var for ERLANG_OTP_VERSION}
        ALPINE_VERSION: ${ALPINE_VERSION? Missing env var for ALPINE_VERSION}
        MIX_ENV: "prod"

  migrate:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    env_file:
      - .env
    command:
      - eval
      - Rumbl.ReleaseTasks.migrate
    healthcheck:
      test: "exit 0"
    depends_on:
      database:
        condition: service_healthy
    networks:
      - default

  app:
    image: ${DOCKER_IMAGE? Missing env var for DOCKER_IMAGE}
    restart: unless-stopped
    env_file:
      - .env
    # Uncomment if not behind Traefik
    #ports:
      #- 0.0.0.0:${PUBLIC_DOMAIN_PORT}:${SERVER_HTTP_PORT}
    networks:
      - traefik
      - default
    depends_on:
      migrate:
        condition: service_started
      database:
        condition: service_healthy
    labels:
      - "traefik.enable=true"
      - "traefik.backend=${PUBLIC_DOMAIN? Missing env var PUBLIC_DOMAIN}"
      - "traefik.docker.network=traefik"
      - "traefik.port=${SERVER_HTTP_PORT}"
      - "traefik.frontend.rule=Host:${PUBLIC_DOMAIN}"

  database:
    image: postgres:10-alpine
    restart: unless-stopped
    healthcheck:
      test: "exit 0"
    env_file:
      - .env
    environment:
      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - ${DATABASE_DIR? Missing env var DATABASE_DIR}/.${APP_NAME}/database:/var/lib/postgresql/data
    networks:
      - default

networks:
  traefik:
    external: true

If you continue to see the problem with migrations with this docker-compose file then you must have some issue in the Kubernetes configuration.

The .env file to be at same level of docker-compose.yml:

# .env

# Adjust as you see fit
DOCKER_IMAGE=public_domain:tag

ELIXIR_VERSION=1.11.3
ERLANG_OTP_VERSION=23.2.2
ALPINE_VERSION=3.12.1

SERVER_HOSTNAME=localhost
SERVER_HTTP_PORT=4000

PUBLIC_SCHEME=https
PUBLIC_DOMAIN=example.com
PUBLIC_DOMAIN_PORT=443

Build the release with:

docker-compose build release

Run the release with:

docker-compose up --detach app

I also use a Docker image based on the official guide, but I have fixed the insecure use of nobody user:

github.com/phoenixframework/phoenix

Deployment guide shouldn't use nobody user in the Dockerfile

opened 11:13AM - 29 Jan 21 UTC

closed 09:48AM - 27 Apr 21 UTC

Exadra37

The [deployment guide](https://github.com/phoenixframework/phoenix/blob/b973280f…33f3403efbab14fa012b53596826d5a1/guides/deployment/releases.md) has a dockerfile example that suggests the use of the `nobody` user, but this user is not recommend to run any programs, it's reserved for use only with NFS, as per the [Ubuntu wiki](https://wiki.ubuntu.com/nobody): > User nobody on a Unix system is traditionally user id 65534. This user is used by NFS servers when they cannot trust the client-supplied uids and gids, or when the root-squash option is being used. > >See https://www.kernel.org/doc/ols/2006/ols2006v2-pages-59-72.pdf and https://www.oreilly.com/openbook/linag2/book/ch14.html for more information on user nobody. > >Some misguided programs or guides suggest that this user should be used for untrusted program execution or handling untrusted data. This is bad advice. Services should have their own, dedicated, user account. Even on sites where NFS is not being used, processes run as user nobody or files owned by user nobody may grant far more privileges than expected, especially if two services have been misconfigured in this fashion. > > **Do not use the user nobody for anything. It is for NFS. ** Using `nobody` as the user can even pose a security risk as explained in the above quote from the Ubuntu wiki. So instead a new and dedicated user should be created. ### Environment N/A ### Expected behavior ```dockerfile FROM hexpm/elixir:1.11.2-erlang-23.1.2-alpine-3.12.1 as build # install build dependencies RUN apk add --no-cache build-base npm git python3 # prepare build dir WORKDIR /app # install hex + rebar RUN mix local.hex --force && \ mix local.rebar --force # set build ENV ENV MIX_ENV=dev # install mix dependencies COPY mix.exs mix.lock ./ RUN mix deps.get --only $MIX_ENV RUN mkdir config # Dependencies sometimes use compile-time configuration. Copying # these compile-time config files before we compile dependencies # ensures that any relevant config changes will trigger the dependencies # to be re-compiled. COPY config/config.exs config/$MIX_ENV.exs config/ RUN mix deps.compile # build assets COPY assets/package.json assets/package-lock.json ./assets/ # install all npm dependencies from scratch RUN npm --prefix ./assets ci --progress=false --no-audit --loglevel=error COPY priv priv # Note: if your project uses a tool like https://purgecss.com/, # which customizes asset compilation based on what it finds in # your Elixir templates, you will need to move the asset compilation step # down so that `lib` is available. COPY assets assets # use webpack to compile npm dependencies - https://www.npmjs.com/package/webpack-deploy RUN npm run --prefix ./assets deploy RUN mix phx.digest # compile and build the release COPY lib lib RUN mix compile # changes to config/runtime.exs don't require recompiling the code COPY config/runtime.exs config/ # uncomment COPY if rel/ exists # COPY rel rel RUN mix release # Start a new build stage so that the final image will only contain # the compiled release and other runtime necessities FROM alpine:3.12.1 AS app RUN apk add --no-cache openssl ncurses-libs ENV USER="phoenix" ENV HOME=/home/"${USER}" ENV APP_DIR="${HOME}/app" # Creates an unprivileged user to be used exclusively to run the Phoenix app RUN \ addgroup \ -g 1000 \ -S "${USER}" && \ adduser \ -s /bin/sh \ -u 1000 \ -G "${USER}" \ -h "${HOME}" \ -D "${USER}" && \ su "${USER}" sh -c "mkdir ${APP_DIR}" # Everything from this line onwards will run in the context of the unprivileged user. USER "${USER}" WORKDIR "${APP_DIR}" COPY --from=build --chown="${USER}":"${USER}" /app/_build/prod/rel/my_app ./ ENTRYPOINT ["bin/my_app"] # Usage: # * build: sudo docker build -t phoenix/my_app . # * shell: sudo docker run --rm -it --entrypoint "" -p 80:4000 -p 443:4040 phoenix/my_app sh # * run: sudo docker run --rm -it -p 80:4000 -p 443:4040 --name my_app phoenix/my_app # * exec: sudo docker exec -it my_app sh # * logs: sudo docker logs --follow --tail 100 my_app # # Extract the production release to your host machine with: # # ``` # mkdir archive # sudo docker run --rm -it --entrypoint "" -v "$PWD/archive:/home/phoenix/archive" phoenix/my_app sh -c "tar zcf /home/phoenix/archive/app.tar.gz ." # ls -al archive # ```` CMD ["start"] ``` Build: ``` sudo docker build -t phoenix/my_app . ``` Check user: ``` $ sudo docker run --rm -it --entrypoint "" phoenix/my_app sh -c "id -u" 1 ↵ 1000 $ sudo docker run --rm -it --entrypoint "" phoenix/my_app sh -c "id -un" phoenix ``` Now the app will run with an unprivileged user that is not used anywhere else in the system as the `nobody` user his. ### Actual behavior ``` $ sudo docker run --rm -it --entrypoint "" phoenix/my_app sh -c "id -u" 65534 $ sudo docker run --rm -it --entrypoint "" phoenix/my_app sh -c "id -un" nobody ``` While `nobody` is still a unprivileged user, its a user that may be used by other parts of the system.

My Dockerfile:

ARG ELIXIR_VERSION
ARG OTP_VERSION
ARG ALPINE_VERSION

FROM hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-alpine-${ALPINE_VERSION} as build

ARG MIX_ENV=prod
ARG BUILD_RELEASE_FROM=master

ENV MIX_ENV=${MIX_ENV}

WORKDIR /app

RUN \
  apk add \
    --no-cache \
    openssh-client \
    build-base \
    npm \
    git \
    python3 && \

  mix local.hex --force && \
  mix local.rebar --force

COPY .env /release/.env
COPY ./.git /workspace

RUN \
  git clone --local /workspace . && \
  git checkout "${BUILD_RELEASE_FROM}" && \

  mix deps.get --only "${MIX_ENV}" && \

  npm --prefix ./assets ci --progress=false --no-audit --loglevel=error && \
  npm run --prefix ./assets deploy && \
  mix phx.digest && \

  mix compile && \
  mix release

# Start a new build stage so that the final image will only contain
# the compiled release and other runtime necessities
FROM alpine:${ALPINE_VERSION} AS app

ENV USER="phoenix"
ENV HOME=/home/"${USER}"
ENV APP_DIR="${HOME}/app"

RUN \
  apk upgrade --no-cache && \
  apk add --no-cache \
    openssl \
    ncurses-libs && \

  # Creates a unprivileged user to run the app
  addgroup \
   -g 1000 \
   -S "${USER}" && \
  adduser \
   -s /bin/sh \
   -u 1000 \
   -G "${USER}" \
   -h "${HOME}" \
   -D "${USER}" && \

  su "${USER}" sh -c "mkdir ${APP_DIR}"

# Everything from this line onwards will run in the context of the unprivileged user.
USER "${USER}"

WORKDIR "${APP_DIR}"

COPY --from=build --chown="${USER}":"${USER}" /app/_build/prod/rel/tasks ./

ENTRYPOINT ["./bin/tasks"]

# Docker Usage:
#  * build: sudo docker build -t phoenix/tasks .
#  * shell: sudo docker run --rm -it --entrypoint "" -p 80:4000 -p 443:4040 phoenix/tasks sh
#  * run:   sudo docker run --rm -it -p 80:4000 -p 443:4040 --env-file .env --name tasks phoenix/tasks
#  * exec:  sudo docker exec -it tasks sh
#  * logs:  sudo docker logs --follow --tail 10 tasks
#
# Extract the production release to your host machine with:
#
# ```
# sudo docker run --rm -it --entrypoint "" --user $(id -u) -v "$PWD/_build:/home/phoenix/_build"  phoenix/tasks sh -c "tar zcf /home/phoenix/_build/app.tar.gz ."
# ls -al _build
# ````
CMD ["start"]

Post #1

benwilson512

Author of Craft GraphQL APIs in Elixir with Absinthe

Can you dump the contents of the migrations table and make sure somehow those rows didn’t get added?

Post #2

terenceponce

Your suggestion made me look back at my Dockerfile line by line and I found out that I wasn’t actually copying the priv folder.

Rookie mistake, but it’s fixed now. I even removed the use of nobody as mentioned in the link that you shared.

Here is the final Dockerfile that I’m using now:

FROM hexpm/elixir:1.11.4-erlang-23.3.1-alpine-3.13.3 AS build
RUN apk add --no-cache build-base npm git python3
WORKDIR /app
RUN mix local.hex --force && \
    mix local.rebar --force
ENV MIX_ENV=prod
COPY mix.exs mix.lock ./
RUN mix deps.get --only $MIX_ENV
RUN mkdir config
COPY config/config.exs config/releases.exs config/$MIX_ENV.exs config/
RUN mix deps.compile
COPY priv priv
COPY lib lib
COPY rel rel
RUN mix do compile, release

FROM alpine:3.13 AS app
RUN apk add --no-cache openssl ncurses-libs
ENV USER="phoenix"
ENV HOME=/home/"${USER}"
ENV APP_DIR="${HOME}/app"
RUN \
  addgroup \
   -g 1000 \
   -S "${USER}" && \
  adduser \
   -s /bin/sh \
   -u 1000 \
   -G "${USER}" \
   -h "${HOME}" \
   -D "${USER}" && \
  su "${USER}" sh -c "mkdir ${APP_DIR}"
USER "${USER}"
WORKDIR "${APP_DIR}"
EXPOSE 4000
COPY --from=build --chown="${USER}":"${USER}" /app/_build/prod/rel/my_app ./
ENTRYPOINT ["bin/my_app"]
CMD ["start"]

Post #3

— All posts loaded —