How to start dynamic supervisor workers on application startup?

Questions / Help
, ,
1

Hello,

In our app, we have to start dynamic amount of workers and we can’t just define them as a part of supervision tree in the application’s start callback, because of using :simple_one_for_one strategy

use Application

 def start(_type, _args) do
    children = [
      supervisor(Registry, [:unique, Postman.AccountRegistry])
    ]
    account_supervisors = Enum.map(accounts(), &account_supervisor/1)

    Supervisor.start_link(children ++ account_supervisors, [strategy: :one_for_one, name: Postman.Supervisor])
  end

  defp account_supervisors(account),
    do: Enum.map(accounts, &(supervisor(Postman.Processor.AccountSupervisor, [&1.name], id: &1.name)))


defmodule Postman.Processor.AccountSupervisor do
  use Supervisor

  def start_link(account_name) do
    Supervisor.start_link(__MODULE__, nil, name: via_tuple(account_name))
  end

  def start_child(account_name, max_demand),
    do: via_tuple(account_name) |> Supervisor.start_child([account_name, max_demand])

  def init(_) do
    supervise(
      [worker(Postman.Processor.Stage.Account, [])],
      strategy: :simple_one_for_one
    )
  end

  defp via_tuple(account_name),
    do: {:via, Registry, {Postman.AccountRegistry, "#{account_name}-supervisor"}}

end

That means to spawn new workers with AccountSupervisor.start_child/2 we have to wait until supervisor finishes its initialization.

The only way around that I can think of is to add GenStage that will start after Supervisors and which solely job will be to spawn children. But is spawning order guaranteed, i.e. is it synchronous?

Maybe there is a whole better way get around that problem?

2 Likes
2

I don’t quite understand your problem, but why can’t you just put a genserver at the end of the children list, which will start all your accounts and terminate? Is there really a need for genstage?

application.ex

defmodule Test.Application do
  use Application

  def start(_type, _args) do
    import Supervisor.Spec, warn: false

    children = [
      supervisor(Registry, [:unique, Test.Registry]),
      supervisor(Test.Account.Supervisor, []),
      worker(Test.Account.Starter, [], restart: :transient)
    ]

    opts = [strategy: :one_for_one, name: Test.Supervisor]
    Supervisor.start_link(children, opts)
  end
end

account/supervisor.ex

defmodule Test.Account.Supervisor do
  use Supervisor

  def start_link do
    Supervisor.start_link(__MODULE__, nil, name: __MODULE__)
  end

  def start_child(account) do
    Supervisor.start_child(__MODULE__, [account])
  end

  def init(_) do
    children = [
      worker(Test.Account, [])
    ]

    supervise(children, strategy: :simple_one_for_one)
  end
end

account/starter.ex

defmodule Test.Starter do
  use GenServer

  def start_link do
    GenServer.start_link(__MODULE__, nil)
  end

  def init(_) do
    send(self(), :start_children)
    {:ok, nil}
  end

  def handle_info(:start_children, _) do
    accounts = [:a, :b, :c]
    Enum.each(accounts, fn account ->
      Test.Account.Supervisor.start_child(account)
    end)
    {:stop, "iz kil"}
  end
end

Anyway, it’s just a suggestion. I also think there is a simpler way.

6 Likes
3

Yup, this is the way to go. You don’t need a GenServer though, a Task would suffice. And you want to pass worker(Test.Starter, [], restart: :transient) so it is not restarted right after it terminates.

9 Likes
4

@idi527 you’re right - it should be GenServer there.

Thanks a lot, folks!

3 Likes
5

I have this use case; an extra genserver works well; but wondering if it could be done via a task

So far (with recent syntax) :

def start(_type, _args) do
    children = [
      {
        SomeSupervisor,
        strategy: :one_for_one
      },
      Supervisor.child_spec({Task, start_stuffs}, id: :start_stuffs, restart: :transient)
    ]

    opts = [strategy: :one_for_one, name: App.Supervisor]
    Supervisor.start_link(children, opts)
  end

  def start_stuffs do
    ["stuff_name1", "stuff_name2"]
    |> Enum.each(fn stuff_name ->
      IO.puts("Processing stuff #{stuff_name}")
      SomeSupervisor.supervise_stuff(stuff_name)
    end)
  end

Ot is there a completely different way now to achieve that ?