How to quickly add to list in map?

That attempt counts nils as 0 instead of skipping them, at least it like this from a first glance, is this really what you want?

Obviously I don’t care about line count - I like code that is easy to change:

defmodule Demo do
  defp initial_total(:response_time, time),
    do: if(is_number(time), do: {time, 1}, else: {0, 0})

  defp initial_total(_key, value),
    do: value || 0

  defp initial_aggregate({key, value}, aggregate),
    do: Map.put(aggregate, key, initial_total(key, value))

  defp merge_total(:response_time, {total_time, count} = total, time),
    do: if(is_number(time), do: {total_time + time, count + 1}, else: total)

  defp merge_total(_key, total, value),
    do: total + (value || 0)

  defp merge_aggregate({key, value}, aggregate) do
    new_total =
      case Map.fetch(aggregate, key) do
        {:ok, total} ->
          merge_total(key, total, value)

        _ ->
          initial_total(key, value)
      end

    Map.put(aggregate, key, new_total)
  end

  defp item_aggregator({key, data}, stats) do
    new_aggregate =
      case Map.fetch(stats, key) do
        {:ok, aggregate} ->
          Enum.reduce(data, aggregate, &merge_aggregate/2)

        _ ->
          Enum.reduce(data, %{}, &initial_aggregate/2)
      end

    Map.put(stats, key, new_aggregate)
  end

  defp finalize_total({:response_time, {total_time, count}}, aggregate) do
    if count > 0 do
      Map.put(aggregate, :response_time, div(total_time, count))
    else
      aggregate
    end
  end

  defp finalize_total({key, total}, aggregate) do
    Map.put(aggregate, key, total)
  end

  defp finalize_aggregate({key, aggregate}, stats),
    do: Map.put(stats, key, Enum.reduce(aggregate, %{}, &finalize_total/2))

  def make_stats(items) do
    items
    |> List.foldl(%{}, &item_aggregator/2)
    |> Enum.reduce(%{}, &finalize_aggregate/2)
  end
end

#
# item: {:group_1, data}
# data: %{timeout: 0, failure: 0, hits: 1, response_time: 100}
# value: associated with a "key" inside the "data" Map
# aggregate: %{failure: 0, hits: 2, response_time: {1100,2}, timeout: 0}
# total: value associated with a "key" inside the "aggregate" Map
# finalized_aggregate: %{failure: 0, hits: 2, response_time: 550, timeout: 0}
# stats: %{feed_key => (finalized_)aggregate}
#

feed_items = [
  {:group_1, %{timeout: 0, failure: 0, hits: 1, response_time: 100}},
  {:group_1, %{timeout: 0, failure: 0, hits: 1, response_time: 1000}},
  {:group_2, %{timeout: 0, failure: 0, hits: 1, response_time: 50}},
  {:group_2, %{timeout: 0, failure: 0, hits: 1, response_time: 2000}},
  {:group_3, %{timeout: 0, failure: 1, hits: 0}},
  {:group_3, %{timeout: 1, failure: 1, hits: 0, response_time: nil}}
]

IO.inspect(Demo.make_stats(feed_items))

$ elixir demo.exs
%{
  group_1: %{failure: 0, hits: 2, response_time: 550, timeout: 0},
  group_2: %{failure: 0, hits: 2, response_time: 1025, timeout: 0},
  group_3: %{failure: 2, hits: 0, timeout: 1}
}

This was my weak attempt, however it’s failing if there’s a single item which has a response_value of nil… which is showing me I have absolutely no idea how this works.. I thought value1 was the existing value from the accumulator and value2 was from the incoming map (from the list)

EDIT OK crap I see it, its because now it’s setting it for every value2…

IO.inspect List.foldl(items, %{}, fn ({feed_name, list_stats}, acc) ->
  Map.update(acc, feed_name, list_stats, fn acc_stats ->
    # handle the first item
    list_stats = %{list_stats | response_time: {list_stats.response_time || 0, 0}}
    Map.merge(acc_stats, list_stats, fn key, value1, value2 ->
      case key do
        :response_time ->
          {value1 + (value2 || 0), value1 + 1}
        _ ->
          (value1 || 0) + (value2 || 0)
      end
    end)
  end)
end)

I’m sure it does not work with your input directly, but if you want to retain the keys of your old data, this should work out:

data
|> Enum.map(fn {k, l} -> {k, Enum.reduce(from above)} end)
|> Map.new()

Sorry for only having such a half snippet, but I’m currently on my mobile.

Does this apply to my input data in post #2 of this thread?

This is what I had before adding averaging which seems to be the simplest way “so far” (thanks to help of peerreynders). I also need to combine it in the final output as well

IO.inspect List.foldl(items, %{}, fn ({date, feed_name, list_stats}, acc) ->
  Map.update(acc, feed_name, list_stats, fn acc_stats ->
    Map.merge(acc_stats, list_stats, fn _key, value1, value2 ->
      # case key do... check if :response_time and then return {total, count} instead of total
      (value1 || 0) + (value2 || 0)
    end)
  end)
end)

Wait? A simple average calculator with a reducer and you have 3 clauses? Then theres something wrong…

{sum, cnt} = Enum.reduce(list, {0, 0}, fn
  nil, acc -> acc
  n, {sum, cnt} -> {sum + n, cnt + 1}
end
if cnt > 0, do: sum / cnt, else: {:error, :no_elements_to_average_over}

This is what I ended up doing… it’s so insanely verbose but I just couldnt’ get the {total, count} method to work I’m hoping it’s not super simple and I just missed something (very likely).

I first calculate the totals in a list, then i calculate the actual list, and then I combine them

feed_items = :queue.to_list(state[:feed_stats])

feed_counts = List.foldl(feed_items, %{}, fn ({feed_name, %{response_time: response_time}}, acc) ->
  count = if response_time, do: 1, else: 0
  Map.update(acc, feed_name, count, fn acc_total -> acc_total + count end)
end)

feed_stats = List.foldl(feed_items, %{}, fn ({feed_name, list_stats}, acc) ->
  # Helpful things to know:
  # - https://forum.elixirforum.com/t/how-to-quickly-add-to-list-in-map/18190
  # - Map.update/4 which runs a function on each value https://hexdocs.pm/elixir/Map.html#update/4
  # - Map.merge/3 which runs a function on each conflicting key https://hexdocs.pm/elixir/Map.html#merge/3
  Map.update(acc, feed_name, list_stats, fn acc_stats ->
    Map.merge(acc_stats, list_stats, fn _key, value1, value2 ->
      (value1 || 0) + (value2 || 0)
    end)
  end)
end) |> Enum.map(fn {feed_name, combined_stats} ->
  total = combined_stats[:response_time]
  feed_count = feed_counts[feed_name]
  stats_with_avg = case feed_count do
    # if there's not a single sucessful request just show -1
    0 -> combined_stats |> Map.put(:response_time, -1)
    _ -> combined_stats |> Map.put(:response_time, round(total / feed_count))
  end
    {feed_name, stats_with_avg}
end) |> Enum.into(%{})

Ok so i ended up thinking this was easy but it turned out to be really complicated because the input value can now be nil, integer, or {total, count}, and now the reducer function can accept any of those as input

I had about 3 guard clauses and I just stopped because it’s too confusing to reason about a month from now

If you are thinking of an easier way would you mind exposing my mind to your greatness?

Dude thats a great idea… ok thank you!

One possible work around is to store {total,count} instead of the average, so when you update you store {total+more,count+1} and when you need the average later you simply calculate total/count.