This proof of concept periodically polls data from a specific API before decoding the JSON. It should create a list of atoms from the binary keys it receives the first time it queries the API for reuse with each subsequent poll.
defmodule ProofOfConcept do
def naive_string_to_atom(raw_json) do
{:ok, decoded_api_response} = Jason.decode(raw_json)
root_keys =
decoded_api_response
|> List.first()
|> Map.keys()
nested_keys =
Enum.flat_map(
root_keys,
fn key ->
decoded_api_response
|> Enum.map(fn map_element ->
foo = Map.get(map_element, key)
case is_map(foo) do
true ->
Map.keys(foo)
false ->
:not_a_key
end
end)
|> Enum.reject(&(&1 == :not_a_key))
end
)
|> Enum.flat_map(& &1)
|> Enum.uniq()
extracted_keys =
(nested_keys ++ root_keys)
|> Enum.map(&String.to_atom(&1))
IO.inspect(extracted_keys, limit: :infinity)
end
def decode(raw_json) do
try do
Jason.decode(raw_json, keys: :atoms!)
rescue
_ ->
IO.puts("Converting JSON keys to atoms . . . ")
naive_string_to_atom(raw_json)
main()
end
end
def main() do
api_url = "https://wavescap.com/api/assets.json"
{:ok, api_response} = HTTPoison.get(api_url)
decode(api_response.body)
end
end
The problem:
Apparently the atomized keys are never retained in memory, requiring them to be recreated each time updated data is retrieved from the API.
iex(1)> ProofOfConcept.main
Converting JSON keys to atoms . . .
[:firstPrice_brln, :firstPrice_btc, :firstPrice_cnyn, :firstPrice_eth,
:firstPrice_eurn, :firstPrice_gbpn, :firstPrice_jpyn, :firstPrice_ngnn,
:firstPrice_rubn, :firstPrice_tryn, :firstPrice_uahn, :"firstPrice_usd-n",
:firstPrice_waves, :lastPrice_brln, :lastPrice_btc, :lastPrice_cnyn,
:lastPrice_eth, :lastPrice_eurn, :lastPrice_gbpn, :lastPrice_jpyn,
:lastPrice_ngnn, :lastPrice_rubn, :lastPrice_tryn, :lastPrice_uahn,
:"lastPrice_usd-n", :lastPrice_waves, :"24h_vol_brln", :"24h_vol_btc",
:"24h_vol_cnyn", :"24h_vol_eth", :"24h_vol_eurn", :"24h_vol_gbpn",
:"24h_vol_jpyn", :"24h_vol_ngnn", :"24h_vol_rubn", :"24h_vol_tryn",
:"24h_vol_uahn", :"24h_vol_usd-n", :"24h_vol_waves", :circulating, :data,
:excludedFromCirculating, :id, :name, :precision, :sender, :shortcode, :start,
:totalSupply, :trades, :website]
Converting JSON keys to atoms . . .
[:firstPrice_brln, :firstPrice_btc, :firstPrice_cnyn, :firstPrice_eth,
:firstPrice_eurn, :firstPrice_gbpn, :firstPrice_jpyn, :firstPrice_ngnn,
:firstPrice_rubn, :firstPrice_tryn, :firstPrice_uahn, :"firstPrice_usd-n",
:firstPrice_waves, :lastPrice_brln, :lastPrice_btc, :lastPrice_cnyn,
:lastPrice_eth, :lastPrice_eurn, :lastPrice_gbpn, :lastPrice_jpyn,
:lastPrice_ngnn, :lastPrice_rubn, :lastPrice_tryn, :lastPrice_uahn,
:"lastPrice_usd-n", :lastPrice_waves, :"24h_vol_brln", :"24h_vol_btc",
:"24h_vol_cnyn", :"24h_vol_eth", :"24h_vol_eurn", :"24h_vol_gbpn",
:"24h_vol_jpyn", :"24h_vol_ngnn", :"24h_vol_rubn", :"24h_vol_tryn",
:"24h_vol_uahn", :"24h_vol_usd-n", :"24h_vol_waves", :circulating, :data,
:excludedFromCirculating, :id, :name, :precision, :sender, :shortcode, :start,
:totalSupply, :trades, :website]
Converting JSON keys to atoms . . .
[:firstPrice_brln, :firstPrice_btc, :firstPrice_cnyn, :firstPrice_eth,
:firstPrice_eurn, :firstPrice_gbpn, :firstPrice_jpyn, :firstPrice_ngnn,
:firstPrice_rubn, :firstPrice_tryn, :firstPrice_uahn, :"firstPrice_usd-n",
:firstPrice_waves, :lastPrice_brln, :lastPrice_btc, :lastPrice_cnyn,
:lastPrice_eth, :lastPrice_eurn, :lastPrice_gbpn, :lastPrice_jpyn,
:lastPrice_ngnn, :lastPrice_rubn, :lastPrice_tryn, :lastPrice_uahn,
:"lastPrice_usd-n", :lastPrice_waves, :"24h_vol_brln", :"24h_vol_btc",
:"24h_vol_cnyn", :"24h_vol_eth", :"24h_vol_eurn", :"24h_vol_gbpn",
:"24h_vol_jpyn", :"24h_vol_ngnn", :"24h_vol_rubn", :"24h_vol_tryn",
:"24h_vol_uahn", :"24h_vol_usd-n", :"24h_vol_waves", :circulating, :data,
:excludedFromCirculating, :id, :name, :precision, :sender, :shortcode, :start,
:totalSupply, :trades, :website]
Converting JSON keys to atoms . . .
[:firstPrice_brln, :firstPrice_btc, :firstPrice_cnyn, :firstPrice_eth,
:firstPrice_eurn, :firstPrice_gbpn, :firstPrice_jpyn, :firstPrice_ngnn,
:firstPrice_rubn, :firstPrice_tryn, :firstPrice_uahn, :"firstPrice_usd-n",
:firstPrice_waves, :lastPrice_brln, :lastPrice_btc, :lastPrice_cnyn,
:lastPrice_eth, :lastPrice_eurn, :lastPrice_gbpn, :lastPrice_jpyn,
:lastPrice_ngnn, :lastPrice_rubn, :lastPrice_tryn, :lastPrice_uahn,
:"lastPrice_usd-n", :lastPrice_waves, :"24h_vol_brln", :"24h_vol_btc",
:"24h_vol_cnyn", :"24h_vol_eth", :"24h_vol_eurn", :"24h_vol_gbpn",
:"24h_vol_jpyn", :"24h_vol_ngnn", :"24h_vol_rubn", :"24h_vol_tryn",
:"24h_vol_uahn", :"24h_vol_usd-n", :"24h_vol_waves", :circulating, :data,
:excludedFromCirculating, :id, :name, :precision, :sender, :shortcode, :start,
:totalSupply, :trades, :website]
Converting JSON keys to atoms . . .
[:firstPrice_brln, :firstPrice_btc, :firstPrice_cnyn, :firstPrice_eth,
:firstPrice_eurn, :firstPrice_gbpn, :firstPrice_jpyn, :firstPrice_ngnn,
:firstPrice_rubn, :firstPrice_tryn, :firstPrice_uahn, :"firstPrice_usd-n",
:firstPrice_waves, :lastPrice_brln, :lastPrice_btc, :lastPrice_cnyn,
:lastPrice_eth, :lastPrice_eurn, :lastPrice_gbpn, :lastPrice_jpyn,
:lastPrice_ngnn, :lastPrice_rubn, :lastPrice_tryn, :lastPrice_uahn,
:"lastPrice_usd-n", :lastPrice_waves, :"24h_vol_brln", :"24h_vol_btc",
:"24h_vol_cnyn", :"24h_vol_eth", :"24h_vol_eurn", :"24h_vol_gbpn",
:"24h_vol_jpyn", :"24h_vol_ngnn", :"24h_vol_rubn", :"24h_vol_tryn",
:"24h_vol_uahn", :"24h_vol_usd-n", :"24h_vol_waves", :circulating, :data,
:excludedFromCirculating, :id, :name, :precision, :sender, :shortcode, :start,
:totalSupply, :trades, :website]
After each new API poll, why aren’t the atoms that were previously generated being reused?