Poison decode struct with improper list

[jeramyRR]

Does anyone have any experience decoding json into structs, with Poison, where the struct has an improper list?

I’m trying to decode json from binance api that looks like the following:

 {
  "timezone": "UTC",
  "serverTime": 1508631584636,
  "rateLimits": [{
      "rateLimitType": "REQUESTS_WEIGHT",
      "interval": "MINUTE",
      "limit": 1200
    },
    {
      "rateLimitType": "ORDERS",
      "interval": "SECOND",
      "limit": 10
    },
    {
      "rateLimitType": "ORDERS",
      "interval": "DAY",
      "limit": 100000
    }
  ],
  "exchangeFilters": [],
  "symbols": [{
    "symbol": "ETHBTC",
    "status": "TRADING",
    "baseAsset": "ETH",
    "baseAssetPrecision": 8,
    "quoteAsset": "BTC",
    "quotePrecision": 8,
    "orderTypes": ["LIMIT", "MARKET"],
    "icebergAllowed": false,
    "filters": [{
      "filterType": "PRICE_FILTER",
      "minPrice": "0.00000100",
      "maxPrice": "100000.00000000",
      "tickSize": "0.00000100"
    }, {
      "filterType": "LOT_SIZE",
      "minQty": "0.00100000",
      "maxQty": "100000.00000000",
      "stepSize": "0.00100000"
    }, {
      "filterType": "MIN_NOTIONAL",
      "minNotional": "0.00100000"
    }]
  }]
}

I have structs defined for each key value pair, and Poison is decoding it like a champ all the way up to the filters inside the symbols list. I believe it is because that data falls into the improper list category. Below is my call to Poison.decode:

@spec decode(json: String.t()) :: %ExchangeInfo{}
def decode(json) do
        json
        |> Poison.decode!(
          as: %ExchangeInfo{
            rateLimits: [%RateLimit{}],
            exchangeFilters: [%ExchangeFilter{}],
            symbols: [
              %Symbol{
                filters: [
                  %IcebergParts{},
                  %LotSize{},
                  %Price{},
                  %MaxNumOrders{},
                  %MaxNumAlgoOrders{},
                  %MinNotional{}
                ]
              }
            ]
          }
        )
end

And here is how I have the Symbol struct defined:

@derive [Poison.Encoder]
  defstruct [
    :symbol,
    :status,
    :baseAsset,
    :baseAssetPrecision,
    :quoteAsset,
    :quotePrecision,
    :orderTypes,
    :icebergAllowed,
    filters: []
  ]

The part that I’m jacking up is the filters section of that decode expression. I’ll spare ya’ll the other structs, but they are pretty simple.

Any ideas on how I can get Poison to decode the filters section into the proper struct?

[ConnorRigby]

This is actually one of my favorite use cases for macros in Elixir. I’ll show how i’d do it without macros first and if there is interest i can show how i’d build a macro later.

Anyway i really don’t like Poison’s as: %Data{} feature. I won’t go into much details, other than you are now locked into using Poison for this feature that other json libraries might not support. Long story short: you should separate you’re JSON decoding from structuring and restructuring. Elixir != JSON and it shouldn’t be treated as if it is. Anyway here’s my decoder:

defmodule ExchangeInfo do
  defmodule RateLimit do
    defstruct [
      :rateLimitType,
      :interval,
      :limit
    ]

    @keys ~w(rateLimitType interval limit)
    def decode(%{} = map) do
      map
      |> Map.take(@keys)
      |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
      |> Enum.map(&decode/1)
      |> fn(data) -> struct(__MODULE__, data) end.()
    end

    def decode({k, v}), do: {k, v}
  end

  defmodule ExchangeFilter do
    @keys ~w()
    defstruct []

    def decode(%{} = map) do
      map
      |> Map.take(@keys)
      |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
      |> Enum.map(&decode/1)
      |> fn(data) -> struct(__MODULE__, data) end.()
    end

    def decode({k, v}), do: {k, v}
  end

  defmodule Symbol do
    defmodule PriceFilter do
      @keys ~w(maxPrice minPrice tickSize)
      defstruct [:maxPrice, :minPrice, :tickSize]
      def decode(%{} = map) do
        map
        |> Map.take(@keys)
        |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
        |> Enum.map(&decode/1)
        |> fn(data) -> struct(__MODULE__, data) end.()
      end

      def decode({k, v}), do: {k, v}
    end

    defmodule LotSize do
      @keys ~w(maxPrice minPrice tickSize)
      defstruct [:maxPrice, :minPrice, :tickSize]
      def decode(%{} = map) do
        map
        |> Map.take(@keys)
        |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
        |> Enum.map(&decode/1)
        |> fn(data) -> struct(__MODULE__, data) end.()
      end

      def decode({k, v}), do: {k, v}
    end

    defmodule MinNotional do
      @keys ~w(minNotional)
      defstruct [:minNotional]
      def decode(%{} = map) do
        map
        |> Map.take(@keys)
        |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
        |> Enum.map(&decode/1)
        |> fn(data) -> struct(__MODULE__, data) end.()
      end

      def decode({k, v}), do: {k, v}
    end

    @keys ~w(symbol status baseAsset baseAssetPrecision quoteAsset quotePrecision orderTypes icebergAllowed filters)
    defstruct [
      :symbol,
      :status,
      :baseAsset,
      :baseAssetPrecision,
      :quoteAsset,
      :quotePrecision,
      :orderTypes,
      :icebergAllowed,
      :filters,
    ]

    def decode(%{} = map) do
      map
      |> Map.take(@keys)
      |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
      |> Enum.map(&decode/1)
      |> fn(data) -> struct(__MODULE__, data) end.()
    end

    def decode({:filters, filters}) when is_list(filters) do
      {:filters, Enum.map(filters, &decode_filters/1)}
    end

    def decode({k, v}), do: {k, v}

    defp decode_filters(%{"filterType" => "PRICE_FILTER"} = data), do: PriceFilter.decode(data)
    defp decode_filters(%{"filterType" => "MIN_NOTIONAL"} = data), do: MinNotional.decode(data)
    defp decode_filters(%{"filterType" => "LOT_SIZE"} = data), do: LotSize.decode(data)
  end

  defstruct [
    :rateLimits,
    :exchangeFilters,
    :symbols
  ]

  @keys ~w(rateLimits exchangeFilters symbols)
  def decode(%{} = map) do
    map
    |> Map.take(@keys)
    |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
    |> Enum.map(&decode/1)
    |> fn(data) -> struct(__MODULE__, data) end.()
  end

  def decode({:rateLimits, limits}) when is_list(limits) do
    {:rateLimits, Enum.map(limits, &RateLimit.decode/1)}
  end

  def decode({:exchangeFilters, filters}) do
    {:exchangeFilters, Enum.map(filters, &ExchangeFilter.decode/1)}
  end

  def decode({:symbols, symbols}) when is_list(symbols) do
    {:symbols, Enum.map(symbols, &Symbol.decode/1)}
  end
end

[jeramyRR]

Thanks for taking the time out to put that together. You’ve got me really curious about the macro now!

[ConnorRigby]

So you may have noticed a lot of duplication in that decoder. Duplication can be a great use case for a macro. Each and every module defined above has a decode/1 function. Lets extract that out. Here’s a refactored module:

defmodule ExchangeInfo do
  import Decoder

  defmodule RateLimit do
    mdecode ~w(rateLimitType interval limit)
  end

  defmodule ExchangeFilter do
    mdecode ~w()
  end

  defmodule Symbol do
    defmodule PriceFilter do
      mdecode ~w(maxPrice minPrice tickSize)
    end

    defmodule LotSize do
      mdecode ~w(maxPrice minPrice tickSize)
    end

    defmodule MinNotional do
      mdecode ~w(minNotional)
    end

    mdecode ~w(symbol status baseAsset baseAssetPrecision quoteAsset quotePrecision orderTypes icebergAllowed filters) do
      mlist :filters, fn(data) ->
        case data do
          %{"filterType" => "PRICE_FILTER"} -> PriceFilter.decode(data)
          %{"filterType" => "MIN_NOTIONAL"} -> MinNotional.decode(data)
          %{"filterType" => "LOT_SIZE"} -> LotSize.decode(data)
        end
      end
    end

  end

  mdecode ~w(rateLimits exchangeFilters symbols) do
    mlist :rateLimits, RateLimit
    mlist :exchangeFilters, ExchangeFilter
    mlist :symbols, Symbol
  end
end

a bit cleaner in my opinion. Here’s the Decoder module implementation:

defmodule Decoder do
  @doc "Defines a struct and decode/1 function."
  defmacro mdecode(str_fields, block \\ [do: nil])

  defmacro mdecode(str_fields, [do: _] = block) do
    quote do
      defstruct Enum.map(unquote(str_fields), &String.to_atom/1)

      def decode(%{} = map) do
        map
        |> Map.take(unquote(str_fields))
        |> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
        |> Enum.map(&decode/1)
        |> fn(data) -> struct(__MODULE__, data) end.()
      end

      unquote(block)

      def decode({k, v}), do: {k, v}
    end
  end

  @doc "Defines a decode/1 function that does an `Enum.map` with a function or module."
  defmacro mlist(field, args) do
    quote do
      cond do
        is_atom(unquote(args)) ->
          def decode({unquote(field), arg}) when is_list(arg) do
            {unquote(field), Enum.map(arg, fn(data) -> unquote(args).decode(data) end)}
          end
        is_function(unquote(args)) ->
          def decode({unquote(field), arg}) when is_list(arg) do
            {unquote(field), Enum.map(arg, fn(data) -> unquote(args).(data) end)}
          end
      end

    end
  end
end

Pretty simple.
mdecode defines a struct and the head decode/1 function from the original implementation.
that Enum.map(&decode/1) will call decode({key, value}) for every one of our expected keys after converting them to atoms. the unquote(block) is a bit of a hack to make the syntax look beteter imo. You could do this without that pretty easily.

the mlist/2 macro just defines that function decode({key, value}) and maps over value by either calling unquote(module).decode/1 or if supplied a function, it calls that instead which is what we do for :filters

After writing that all up i realized your original question wasn’t really answered and this may have been an overload of information depending on how comfortable with Elixir/Metaprogramming you are.

The main point i guess i want to get across is a subjective one: don’t just take my word for it -
Don’t rely on features of a dep outside of the scope of it’s domain. Poison is a json decoder, not a domain data structurer/destructurer. It’s main concern should be taking binary data and parsing it as Elixir data, NOT taking binary data and turning it into youre domain specific data. It doesn’t have enough information to perform that task.

[jeramyRR]

Thanks again for spending so much time to go through this.