Most efficient way to retrieve multiple values from nested map

Questions / Help
1

Let’s say I have nested map like this :

data = %{
    gps: %{
        lat: 48.857144,
        lon: 2.340242,
        altitude: 35.0
    },
    metadata: %{
        payload: %{
            content: "1aff4c0002154a080bff4c0010774058308876aa3a29",
            scantime: 1644362438
        }
        macaddress: "B8:C0:78:CD:12:AB",
        receivetime: 1663000169,
        tags: "v1.2,test.app"
    }
}

# There is way more depth and values but I will keep it simple here.

What would be the best way to put lat, lon, scantime & receivetime in a new map ? The idea behind this would be to use this new simplified map with Ecto to add rows in database. I also need to ensure that every needed values is set.

Currently I am using something like this :

with {:ok, gps} <- Map.fetch(data, :gps),
     {:ok, lat} <- Map.fetch(gps, :lat),
     {:ok, lon} <- Map.fetch(gps, :lon),
     {:ok, metadata} <- Map.fetch(data, :metadata),
     {:ok, payload} <- Map.fetch(metadata, :payload),
     {:ok, scantime} <- Map.fetch(payload, :scantime),
     ... do
        %{
            lat: lat,
            lon: lon,
            scantime: scantime,
            ...
        }
     end

I need to extract something like twelve values from even more nested map, this looks really awful. Any idea?

2

What should happen if a value is missing? Is that a case you need to worry about, or is the shape of the payload reliable?

1 Like
3

Here you go:

defmodule Example do
  def sample(acc \\ %{}, data, info)

  # in case nested value does not exists
  def sample(acc, nil, _info), do: acc

  # instead of above you may want to use another code
  # as it would place a nil value for each nested info who does not exists in data you passed
  # 
  # def sample(acc, nil, info) when is_atom(info), do: Map.put(acc, info, nil)
  # def sample(acc, nil, info) when is_list(info), do: Enum.reduce(info, acc, &sample(&2, nil, &1))
  # 
  # def sample(acc, nil, info) when is_map(info) do
  #   Enum.reduce(info, acc, &sample(&2, nil, elem(&1, 1)))
  # end

  # when we need to fetch a flat list of fields
  # or said list + some extra nested fields
  def sample(acc, data, info) when is_list(info) do
    # there should be no more than one map in info list
    # the map info stores information for nested fields
    # the remaining items is a list of fields to take from current data
    groups = Enum.group_by(info, &is_map/1)
    [map_info] = groups[true] || [%{}]
    groups[false] |> Enum.reduce(acc, &Map.put(&2, &1, data[&1])) |> sample(data, map_info)
  end

  # here we are reducing info map over our acc
  # which means all nested fields logic goes here
  def sample(acc, data, info) when is_map(info) do
    Enum.reduce(info, acc, fn {info_key, info_value}, acc ->
      sample(acc, data, info_key, info_value)
    end)
  end

  # this clause would match if we want to fetch just one nested field
  defp sample(acc, data, info_key, info_value) when is_atom(info_value) do
    Map.put(acc, info_value, get_in(data, [info_key, info_value]))
  end

  # in any other case we have a map or list which we already support
  # so all we need to do is to call the same logic, but with nested data
  defp sample(acc, data, info_key, info_value) when is_list(info_value) or is_map(info_value) do
    sample(acc, data[info_key], info_value)
  end
end

data = %{
  gps: %{
    lat: 48.857144,
    lon: 2.340242,
    altitude: 35.0
  },
  metadata: %{
    payload: %{
      content: "1aff4c0002154a080bff4c0010774058308876aa3a29",
      scantime: 1_644_362_438
    },
    macaddress: "B8:C0:78:CD:12:AB",
    receivetime: 1_663_000_169,
    tags: "v1.2,test.app"
  }
}

info = %{gps: [:lat, :lon], metadata: [%{payload: :scantime}, :receivetime], a: %{b: :c}}
iex> Example.sample(data, info)

This code would automatically pick data you need by simply passing an info. Also if you want to add something to result you simply can pass it as first argument which means that you can call this function multiple times for different data:

data1
# without passing acc
|> Example.sample(info1)
# with acc (result of above pipe)
|> Example.sample(data2, info2)

Helpful resources:

  1. is_atom/1, is_list/1 and is_map/1 guards
  2. Enum.reduce/3
  3. Map.put/3
3 Likes
4

You can also have a look at a library like GitHub - hissssst/pathex: Fastest way to access data in Elixir.

2 Likes
5

Works like a charm thank you ! I updated it a bit so it will raise an error instead of putting a nil value.

An error should be raised. If you know another way to do it, I would be very interested in seeing it too!

6

In this case, pattern match!

%{
    gps: %{
        lat: lat,
        lon: lon,
        altitude: alt
    },
    metadata: %{
        payload: %{
            scantime: scantime
        }
        receivetime: receive_time,
    } = data
}

Pattern matching is super useful here because it provides a way to declaratively extract what you want.

6 Likes
7

I think I don’t get what you are trying to explain. This is only validating data, right ? How is this going to “flatten” the map ?

8

The pattern matching is not useful only for validation, but also for assigning variables. Simplest example:

iex> %{a: a} = %{a: 5}
%{a: 5}
iex> a
5

What @benwilson512 suggested is to do exactly same for nested structures, for example:

iex> %{nested: %{a: a}, b: b} = %{nested: %{a: 5}, b: 10}
%{nested: %{a: 5}, b: 10}
iex> %{a: a, b: b}
%{a: 5, b: 10}

However since you said that you have many cases for this and they could be even more complicated, the pattern matching here for just a single data structure would take lots of lines. For me it looks much worse comparing to usage of a simple info variable in my example.

1 Like
9

I understand now ! Thank you :slightly_smiling_face:

1 Like
10

I have another question regarding the solution you gave. Is that possible to have a non nested value at the root of the map? Like this :

data = %{
  gps: %{
    lat: 48.857144,
    lon: 2.340242,
    altitude: 35.0
  },
  macaddress: "B8:C0:78:CD:12:AB"
}

I am having a hard time finding a way to retrieve macaddress.

11

Sure thing! Look that in original version I’m using map as a root of input, but in code you would find that also list is supported. The proper info for this data would be:

data = %{
  gps: %{
    lat: 48.857144,
    lon: 2.340242,
    altitude: 35.0
  },
  macaddress: "B8:C0:78:CD:12:AB"
}

info = [%{gps: [:lat, :lon]}, :macaddress]
iex> Example.sample(data, info)

However I found one thing. In original code list part is not best. I would rewrite it to tuple instead, for example:

defmodule Example do
  def sample(acc \\ %{}, data, info)

  # in case nested value does not exists
  def sample(acc, nil, _info), do: acc

  # instead of above you may want to use another code
  # as it would place a nil value for each nested info who does not exists in data you passed
  # 
  # def sample(acc, nil, info) when is_atom(info), do: Map.put(acc, info, nil)
  # def sample(acc, nil, info) when is_list(info), do: Enum.reduce(info, acc, &sample(&2, nil, &1))
  # 
  # def sample(acc, nil, info) when is_map(info) do
  #   Enum.reduce(info, acc, &sample(&2, nil, elem(&1, 1)))
  # end

  # when we want to fetch nested fields and root fields at the same time
  # we are passing {nested_fields_map_info, root_fields_list_info}
  def sample(acc, data, {map_info, list_info}) when is_map(map_info) and is_list(list_info) do
    acc |> sample(data, map_info) |> sample(data, list_info)
  end

  # when we need to fetch a flat list of fields
  def sample(acc, data, info) when is_list(info) do
    Enum.reduce(info, acc, &Map.put(&2, &1, data[&1]))
  end

  # here we are reducing info map over our acc
  # which means all nested fields logic goes here
  def sample(acc, data, info) when is_map(info) do
    Enum.reduce(info, acc, fn {info_key, info_value}, acc ->
      sample(acc, data, info_key, info_value)
    end)
  end

  # this clause would match if we want to fetch just one nested field
  # %{nested: :field}
  defp sample(acc, data, info_key, info_value) when is_atom(info_value) do
    Map.put(acc, info_value, get_in(data, [info_key, info_value]))
  end

  # in any other case call the same logic, but with nested data and info
  # %{nested: [:field1, :field2, …]}
  # or
  # %{nested: %{nested_level2: …}}
  # or
  # %{nested: {%{nested_level2: …}, [:field]}
  defp sample(acc, data, info_key, info_value), do: sample(acc, data[info_key], info_value)
end

data = %{
  gps: %{
    lat: 48.857144,
    lon: 2.340242,
    altitude: 35.0
  },
  metadata: %{
    payload: %{
      content: "1aff4c0002154a080bff4c0010774058308876aa3a29",
      scantime: 1_644_362_438
    },
    macaddress: "B8:C0:78:CD:12:AB",
    receivetime: 1_663_000_169,
    tags: "v1.2,test.app"
  }
}

info = %{gps: [:lat, :lon], metadata: {%{payload: :scantime}, [:receivetime]}}
iex> Example.sample(data, info)

data2 = %{
  gps: %{
    lat: 48.857144,
    lon: 2.340242,
    altitude: 35.0
  },
  macaddress: "B8:C0:78:CD:12:AB"
}

iex> Example.sample(data2, {%{gps: [:lat, :lon]}, [:macaddress]})

This way instead of [%{nested: …}, :field1, …] we have {%{nested: …}, [:field1, …]} which separates nested map info from root list info. This way we no longer need to call Enum.groiup_by/2.

1 Like