GenServer error in prod

Questions / Help
1 
** (stop) bad return value: {:EXIT, {:undef, [{:erts_internal, :abort_pending_connection, [:"my_project@127.0.0.1", {1, #Reference<0.2743369906.2933784582.203476>}], []}, {:net_kernel, :pending_nodedown, 5, [file: 'net_kernel.erl', line: 1040]}, {:net_kernel, :conn_own_exit, 3, [file: 'net_kernel.erl', line: 960]}, {:net_kernel, :do_handle_exit, 3, [file: 'net_kernel.erl', line: 928]}, {:net_kernel, :handle_exit, 3, [file: 'net_kernel.erl', line: 923]}, {:gen_server, :try_dispatch, 4, [file: 'gen_server.erl', line: 637]}, {:gen_server, :handle_msg, 6, [file: 'gen_server.erl', line: 711]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}
Last message: {:EXIT, #PID<0.103.0>, :shutdown}
** (exit) exited in: :gen_server.call(:net_kernel, {:connect, :hidden, :"my_project@127.0.0.1"}, :infinity)
    ** (EXIT) bad return value: {:EXIT, {:undef, [{:erts_internal, :abort_pending_connection, [:"my_project@127.0.0.1", {1, #Reference<0.2743369906.2933784582.203476>}], []}, {:net_kernel, :pending_nodedown, 5, [file: 'net_kernel.erl', line: 1040]}, {:net_kernel, :conn_own_exit, 3, [file: 'net_kernel.erl', line: 960]}, {:net_kernel, :do_handle_exit, 3, [file: 'net_kernel.erl', line: 928]}, {:net_kernel, :handle_exit, 3, [file: 'net_kernel.erl', line: 923]}, {:gen_server, :try_dispatch, 4, [file: 'gen_server.erl', line: 637]}, {:gen_server, :handle_msg, 6, [file: 'gen_server.erl', line: 711]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}
    (stdlib 3.12.1) gen_server.erl:223: :gen_server.call/3
    (distillery 2.1.1) lib/distillery/releases/runtime/control.ex:1006: Distillery.Releases.Runtime.Control.hidden_connect/1
    (distillery 2.1.1) lib/distillery/releases/runtime/control.ex:438: Distillery.Releases.Runtime.Control.ping/2
    (distillery 2.1.1) lib/entry.ex:47: Distillery.Releases.Runtime.Control.main/1
    (stdlib 3.12.1) erl_eval.erl:680: :erl_eval.do_apply/6
    (elixir 1.10.3) lib/code.ex:341: Code.eval_string_with_error_handling/3

Occurs after executing ping or stop in the prod environment, the release process doesn’t notify any errors.

  • Elixir version: 1.10.3

  • Erlang version: 22.3.3

2

This is what I have found on rebar3 discussions:

The most direct guess is coming from the error message you get ( undef is an error when you usually do not have the right module or function to do a given task). That one would point towards probably your prod release not having all the modules it requires to do its task at runtime compared to the dev release.

(…)

Stopped everything, remade a prod release, retested with io:format("~p~n", [lists:sort(code:all_loaded())]) to check the loaded modules and then… everything ran ok. Don’t understand why it didn’t work. Thanks for having answered my question. Sorry for the inconvenience. If the problem appears again and I manage to solve it, I will update the post.

Source: https://www.rebar3.org/discuss/5e4bf0ff88c0fe0018c9999a

Looks like somehow your release is bad and this may be because of incorrect mix.exs and/or distillery configuration or the release process itself. Firstly try to do same as in mentioned reply i.e. stop everything and make a prod release again.

The other things you can do yourself is trying to reproduce it on latest versions i.e. Elixir version 1.10.4 and Erlang version 22.4.9 or even 23.0.3.

Otherwise it would be helpful for me and others if we would have small example project containing only code needed to reproduce issue. If that’s not possible it may be worth to share your redacted mix.exs and release configuration.

1 Like
3

Thank you for your answer. I will revise the modules and try with the latest versions. meanwhile, follow the release config and the mix.exs

rel/config.exs

Path.join(["rel", "plugins", "*.exs"])
|> Path.wildcard()
|> Enum.map(&Code.eval_file(&1))

use Distillery.Releases.Config,
    default_release: :default,
    default_environment: Mix.env()


environment :dev do

  set dev_mode: true
  set include_erts: false
  set cookie: :crypto.hash(:sha256, System.get_env("COOKIE")) |> Base.encode16 |> String.to_atom
end

environment :prod do
  set include_erts: true
  set include_src: false
  set cookie: :crypto.hash(:sha256, System.get_env("COOKIE")) |> Base.encode16 |> String.to_atom
end



release :project_name do
  set version: current_version(:project)
  set applications: [
    :runtime_tools
  ]
end

release :staging do
  set vm_args: "rel/vm.args"
  set version: current_version(:project)
  set applications: [
    :runtime_tools,
    :project_name
  ]
  set commands: [
    migrate: "rel/commands/migrate.sh",
    seed: "rel/commands/seed.sh",
  ]
end

mix.exs

defmodule Project.Mixfile do
  use Mix.Project

  def project do
    [
      app: :Project,
      version: "0.0.1",
      elixir: "~> 1.4",
      elixirc_paths: elixirc_paths(Mix.env),
      compilers: [:phoenix, :gettext, :rustler] ++ Mix.compilers,
      start_permanent: Mix.env == :prod,
      aliases: aliases(),
      deps: deps(),
      rustler_crates: rustler_crates(),
    ]
  end

  def application do
    [
      mod: {project.Application, []},
      extra_applications: [:logger, :runtime_tools, :bamboo, :bamboo_smtp, :soap, :cachex, :httpoison, :poison]
    ]
  end

  defp elixirc_paths(:test), do: ["lib", "test/support"]
  defp elixirc_paths(_),     do: ["lib"]

  defp deps do
    [
      {:phoenix, "~> 1.3.3"},
      {:phoenix_pubsub, "~> 1.0"},
      {:phoenix_ecto, "~> 4.1"},
      {:postgrex, ">= 0.0.0"},
      {:phoenix_html, "~> 2.10"},
      {:phoenix_live_reload, "~> 1.0", only: :dev},
      {:gettext, "~> 0.11"},
      {:cowboy, "~> 1.0"},
      {:plug_cowboy, "~> 1.0"},
      {:distillery, "~> 2.1"},
      {:poison, "~> 3.1"},
      {:rustler, "~> 0.21"},
      {:csv, "~> 2.3.1"},
      {:bcrypt_elixir, "~> 1.0.0"},
      {:brcpfcnpj, "~> 0.1.0"},
      {:con_cache, "~> 0.13.0"},
      {:temp, "~> 0.4"},
      {:sweet_xml, "~> 0.3"},
      {:benchwarmer, "~> 0.0.2" },
      {:elixir_uuid, "~> 1.2" },
      {:file_info, "~> 0.0.2"},
      {:mimetype_parser, "~> 0.1.2"},
      {:elixlsx, "~> 0.3.1"},
      {:httpoison, "~> 1.4"},
      {:number, "~> 1.0"},
      {:bamboo, "~> 1.1"},
      {:bamboo_smtp, "~> 1.7.0"},
      {:soap, "~> 1.0"},
      {:xml_builder, "~> 2.1.1"},
      {:pdf_generator, ">=0.6.0"},
      {:quantum, "~> 2.3"},
      {:timex, "~> 3.0"},
      {:cachex, "~> 3.2"},
      {:html_entities, "~> 0.4"},
      {:plug_crypto, "~> 1.0"},
      {:ex_crypto, "~> 0.10.0"},
      {:pdf2htmlex, "~> 0.2.0"},
      {:floki, "~> 0.26.0"},
      {:logger_file_backend, "~> 0.0.6"},
      {:ecto_sql, "~> 3.4"},
      {:remote_ip, "~> 0.2.0"}
    ]
  end


  defp aliases do
    [
      "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
      "ecto.reset": ["ecto.drop", "ecto.setup"],
      test: ["ecto.create --quiet", "ecto.migrate", "test"]
    ]
  end

  defp rustler_crates do
    [
      project: [
        path: "native/project",
        mode: (if Mix.env == :prod, do: :release, else: :debug),
      ]
    ]
  end
end
4

I have not touched releases for some time, so maybe I’m missing something, but for me both files looks good.

Maybe debugging loaded modules (if possible) would give us some hint …

:code.all_loaded() |> Enum.find(& elem(&1, 0) == :net_kernel) |> is_nil() |> IO.inspect(label: ":net_kernel lookup")
# or simpler:
:code.all_loaded() |> IO.inspect(label: "loaded modules", limit: :infinity)

For my local :dev installation by :asdf it looks like:

iex(1)> :code.all_loaded |> Enum.find(& elem(&1, 0) == :net_kernel)
{:net_kernel,
 '$HOME/.asdf/installs/erlang/23.0.3/lib/kernel-7.0/ebin/net_kernel.beam'}
# Note: $HOME is redacted - don't worry about it :-)

You should have something like:

{:net_kernel, 'redacted-path-to-erts/lib/kernel-7.0/ebin/net_kernel.beam'}
1 Like