Behaviours, defoverridable and implementations - reloaded

I was just looking for documentation to build a Behaviour with some defaults.
I stumbled upon this thread:

As it seems, that’s still true in 2021. I did not find any documentation (except for the basics).

So I had a look at gen_server.ex and put a sample together (see below, see the tests for how I understand things). It does what I expect, but I have some questions:

  • am I missing some important technique?
  • (Solved with module attributes) the way I access the opts seems a little odd…!?
  • (Solved by adding @impl inside using) I get this warning I can’t figure out:

warning: module attribute @impl was not set for function foo/0 callback (specified in MyBehaviour). This either means you forgot to add the “@impl true” annotation before the definition or that you are accidentally overriding this callback
lib/user.ex:2: User (module)

my_behaviour.ex

defmodule MyBehaviour do
  @callback foobar(foo_arg :: any()) :: any()

  @callback foo() :: any()
  @callback bar(bar_arg :: any()) :: any()
  @callback baz(baz_arg1 :: any(), baz_arg2 :: any()) :: any()

  @optional_callbacks foo: 0,
                      bar: 1,
                      baz: 2

  defmacro __using__(opts) do
    quote location: :keep, bind_quoted: [opts: opts] do
      @behaviour MyBehaviour
      @foo_opt Keyword.get(opts, :foo, :no_foo_opt)
      @bar_opt Keyword.get(opts, :bar, :no_bar_opt)
      @baz_opt Keyword.get(opts, :baz, :no_baz_opt)

      @impl true
      def foo() do
        {:foo, @foo_opt}
      end

      @impl true
      def bar(bar_arg) when is_atom(bar_arg) do
        {:bar, bar_arg, @bar_opt}
      end

      @impl true
      def baz(baz_arg1, baz_arg2) do
        {:baz, baz_arg1, baz_arg2, @baz_opt}
      end

      defoverridable baz: 2, foo: 0
    end
  end

  def behaviour_fun(behaviour_fun_arg) do
    {:behaviour_fun, behaviour_fun_arg}
  end
end

user.ex

defmodule User do
  use MyBehaviour, bar: :opt_for_bar, foo: :opt_for_foo

  @impl true
  def bar(arg) do
    {:bar_implemented, arg}
  end

  @impl true
  def baz(arg1, arg2) do
    {:baz_implemented, arg1, arg2}
  end

  @impl true
  def foobar(arg) do
    MyBehaviour.behaviour_fun(arg)
  end
end

test.exs

defmodule Test do
  use ExUnit.Case

  test "default impl is called and option used" do
    assert {:foo, :opt_for_foo} = User.foo()
  end

  test "default impl is called when it matches" do
    assert {:bar, :some_atom, :opt_for_bar} = User.bar(:some_atom)
  end

  test "user impl is called when default doesn't match" do
    assert {:bar_implemented, 1} = User.bar(1)
  end

  test "user impl is always called with defoverrideable" do
    {:baz_implemented, 1, 2} = User.baz(1, 2)
  end

  test "can define function in behaviour" do
    {:behaviour_fun, 1} = User.foobar(1)
  end
end
1 Like

For reading the options, put them in an attribute. Btw, José published an answer related to this Unquoute not working for maps even with Macro.escape - #2 by josevalim

1 Like

the warning regarding the usage of @impl is because you forgot to set it inside __using__.
If you do set it in __using__ and do not set it in your implementation, the compiler will not warn, as it realizes the one inside using is generated code by a macro.

1 Like

OK, no warning when I add @impl inside __using__

defmacro __using__(opts) do
    quote location: :keep, bind_quoted: [opts: opts] do
      ...
      @impl true
      def foo() do
         ...

How does GenServer get away with not doing this?

opts-problem is also solved.

quote location: :keep, bind_quoted: [opts: opts] do
      @behaviour MyBehaviour
      ...
      @foo_opt Keyword.get(opts, :foo, :no_foo_opt)

      def foo() do
        {:foo, @foo_opt}
      end
      ...

thank you!

1 Like

Good question. I will look into it when I have some time to spare

1 Like

I have looked into it.
Here is a reduced case:

defmodule MyBehaviour do
  @callback foo(term) :: any()
  @callback bar(bar_arg :: any()) :: any()

  defmacro __using__(_opts) do
    quote do
      @behaviour MyBehaviour
      def foo(_arg), do: :using

      def bar(arg) when is_atom(bar_arg), do: :using
    end
  end
end

defmodule MyImplementation do
  use MyBehaviour

  @impl true
  def bar(_arg), do: :ok
end

What happens in that you are definining some functions with __using__ but not all of them are overridable. So in the using context, you do not annotate your functions with @impl but in the implementation you do, so you have a function (bar/1 in your original example) that later gets annotated with @impl in the implementation. You are mixing clauses from two different contexts and there for the warning.

The solution is to add bar: 1 to defoverridable

1 Like

This means I have to set defoverridable for all callbacks. So this is not possible:

 test "default impl is called when it matches" do
  assert {:bar, :some_atom, :opt_for_bar} = User.bar(:some_atom)
end

test "user impl is called when default doesn't match" do
  assert {:bar_implemented, 1} = User.bar(1)
end

i.e. a default impl without defoverridable is called if it matches (here bar only matches atom-args) and the user-impl is called otherwise.

Also GenServer does not set defoverridable for all callbacks:

defoverridable child_spec: 1
...
defoverridable code_change: 3, terminate: 2, handle_info: 2, handle_cast: 2, handle_call: 3
@callback init(init_arg :: term) :: ...
@callback handle_call(request :: term, from, state :: term) :: ... # defoverridable
@callback handle_cast(request :: term, state :: term) :: ... # defoverridable
@callback handle_info(msg :: :timeout | term, state :: term) :: ... # defoverridable
@callback handle_continue(continue :: term, state :: term) :: ...
@callback terminate(reason, state :: term) :: term # defoverridable
@callback code_change(old_vsn, state :: term, extra :: term) :: ... # defoverridable
@callback format_status(reason, pdict_and_state :: list) :: term

@optional_callbacks code_change: 3,
                      terminate: 2,
                      handle_info: 2,
                      handle_cast: 2,
                      handle_call: 3,
                      format_status: 2,
                      handle_continue: 2

init is set to be overridable.
format_status and handle_continue do not have a default implementation.

I think that pattern is good, if you want to raise on a specific pattern or type being passed, but other than that is it too much magic to define some clauses inside __using__ in another module, and maybe another library, and to define other clauses in the implementation.

Just to be clear, what I am saying is that you need to set defoverridable for every callback you are defining inside __using__, not for every callback in the behaviour. Because if you do not do this, and you do not annotate them with @impl, when you do annotate your implementations with @impl you will get a warning for every callback that you defined in __impl__ with no @impl, as it happened in your original post.

2 Likes

OK, thanks all clear now!

I’m marking this as solution for future reference, but all credit to @eksperimental!

Summary of what I learned:

  1. There are 4 types of functions related to Behaviour-modules
    1.1 fun_without_default - mandatory callbacks without default implementation, they need a @callback. They have to be implemented in the User-module.
    1.2 fun_with_default - mandatory callbacks with default implementation inside __using__. They need a @callback and have to marked defoverridable. You can’t forbid to override a function, it kinda works with some interesting effects (see above) but will cause a warning. They may be implemented in the User-module, if not, the default is used.
    1.3 optional_fun - optional callback, they need a @callback and have to be marked in optional_callbacks. They may be implemented in the User-module, if not, its an error if they are called.
    1.4 behaviour_fun - functions implemented in the Behaviour-module but not inside __using__.

  2. To access options
    2.1 use quote with bind_quoted
    2.2 put the options into a module attribute inside __using__

defmodule MyBehaviour do
  @callback fun_with_default() :: any()
  @callback fun_without_default() :: any()
  @callback optional_fun() :: any()

  @optional_callbacks optional_fun: 0

  defmacro __using__(opts) do
    quote location: :keep, bind_quoted: [opts: opts] do
      @behaviour MyBehaviour

      @my_opt Keyword.fetch!(opts, :my_opt)

      def fun_with_default() do
        {:fun_with_default, :default, @my_opt}
      end

      defoverridable fun_with_default: 0
    end
  end

  def behaviour_fun() do
    :behaviour_fun
  end
end
defmodule User1 do
  use MyBehaviour, my_opt: :my_opt_value

  @impl true
  def fun_without_default(), do: {:fun_without_default, :implemented}

  def fun_that_just_calls_behaviour_fun(), do: MyBehaviour.behaviour_fun()
end

defmodule User2 do
  use MyBehaviour, my_opt: :my_opt_value

  @impl true
  def fun_without_default(), do: {:fun_without_default, :implemented}

  @impl true
  def fun_with_default(), do: {:fun_with_default, :implemented}

  @impl true
  def optional_fun(), do: {:optional_fun, :implemented}
end
assert {:fun_without_default, :implemented} = User1.fun_without_default()
assert {:fun_with_default, :default, :my_opt_value} = User1.fun_with_default()
assert :behaviour_fun = User1.fun_that_just_calls_behaviour_fun()
assert_raise(UndefinedFunctionError, fn -> User1.optional_fun() end)

assert {:fun_with_default, :implemented} = User2.fun_with_default()
assert {:optional_fun, :implemented} = User2.optional_fun()
8 Likes

This example was very helpful.

Would this be the “best” way to be able to have default functions that can be partially overridden?

defmodule MyBehaviour do
  @callback fun_with_default(name :: atom) :: any()
  @callback fun_without_default() :: any()
  @callback optional_fun() :: any()

  @optional_callbacks optional_fun: 0

  defmacro __using__(opts) do
    quote location: :keep, bind_quoted: [opts: opts] do
      @behaviour MyBehaviour

      @my_opt Keyword.fetch!(opts, :my_opt)

      def fun_with_default(name) do
        {:fun_with_default, :default, @my_opt, name}
      end

      defoverridable fun_with_default: 1
    end
  end

  def behaviour_fun() do
    :behaviour_fun
  end
end

defmodule User1 do
  use MyBehaviour, my_opt: :my_opt_value

  @impl true
  def fun_without_default(), do: {:fun_without_default, :implemented}

  def fun_that_just_calls_behaviour_fun(), do: MyBehaviour.behaviour_fun()
end

defmodule User2 do
  use MyBehaviour, my_opt: :my_opt_value

  @impl true
  def fun_without_default(), do: {:fun_without_default, :implemented}

  @impl true
  def fun_with_default(name), do: {:fun_with_default, :implemented, name}

  @impl true
  def optional_fun(), do: {:optional_fun, :implemented}
end

defmodule User3 do
  use MyBehaviour, my_opt: :my_opt_value

  @impl true
  def fun_without_default(), do: {:fun_without_default, :implemented}

  @impl true
  def fun_with_default(:overridden), do: {:fun_with_default, :implemented, :overridden}
  def fun_with_default(name), do: super(name)
end