Parsing Maps Quickly and Efficiently

Questions / Help
1 
data = [
  %{
    asset: "51LxAtwBXapvvTFSbbh4nLyWFxH6x8ocfNvrXxbTChze",
    balance: 6.0e-6,
    name: "TORCorp",
    priced_in: %{
      "WAVES" => %{
        asset: "51LxAtwBXapvvTFSbbh4nLyWFxH6x8ocfNvrXxbTChze",
        last_price: 3.65e-4,
        priced_in: "WAVES",
        volume: 1998.66525750942
      }
    }
  },
  %{
    asset: "GAAZu8fsWfrZBFgweMMAFEcjrW9CFRxPEqmGcbSGWVDF",
    balance: 3.0,
    name: "K975",
    priced_in: %{
      "WAVES" => %{
        asset: "GAAZu8fsWfrZBFgweMMAFEcjrW9CFRxPEqmGcbSGWVDF",
        last_price: 3.9735e-4,
        priced_in: "WAVES",
        volume: 19.3305527100074
      }
    }
  },
  %{
    asset: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
    balance: 0.05218729,
    name: "WETH",
    priced_in: %{
      "34N9YcEETLWn93qYQ64EsP1x89tSruJU44RrEMSXXEPJ" => %{
        asset: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        last_price: 260,
        priced_in: "34N9YcEETLWn93qYQ64EsP1x89tSruJU44RrEMSXXEPJ",
        volume: 33.2756821322367
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        last_price: 0.02615052,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 26781.2825591284
      },
      "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p" => %{
        asset: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        last_price: 232.6,
        priced_in: "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
        volume: 68058.0649281976
      },
      "WAVES" => %{
        asset: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        last_price: 194.85480268,
        priced_in: "WAVES",
        volume: 72328.2137156855
      }
    }
  },
  %{
    asset: "81HnQ1PWViWUiv39NGkPjbfDehW1edjeJe6XxVhw1TPZ",
    balance: 1.0,
    name: "MGOLD",
    priced_in: %{ 
      "WAVES" => %{
        asset: "81HnQ1PWViWUiv39NGkPjbfDehW1edjeJe6XxVhw1TPZ",
        last_price: 0.012,
        priced_in: "WAVES",
        volume: 223.52763
      }
    }
  },
  %{
    asset: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
    balance: 0.00758563,
    name: "WBTC",
    priced_in: %{
      "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p" => %{
        asset: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        last_price: 8880.89,
        priced_in: "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
        volume: 146370.427831741
      },
      "Ft8X1v1LTa1ABafufpaCWyVj8KkaxUWE6xBhW6sNFJck" => %{
        asset: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        last_price: 17400,
        priced_in: "Ft8X1v1LTa1ABafufpaCWyVj8KkaxUWE6xBhW6sNFJck",
        volume: 90.7845787213622
      },
      "Gtb1WRznfchDnTh37ezoDTJ4wcoKaRsKqKjJjy7nm2zU" => %{
        asset: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        last_price: 15751,
        priced_in: "Gtb1WRznfchDnTh37ezoDTJ4wcoKaRsKqKjJjy7nm2zU",
        volume: 4046.63781370651
      }
    }
  },
  %{
    asset: "EbLVSrAi6vS3AkLwBinzZCvAXP2yYiFJEzj1MBVHcwZ5",
    balance: 5.0e-8,
    name: "ASIMI",
    priced_in: %{
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "EbLVSrAi6vS3AkLwBinzZCvAXP2yYiFJEzj1MBVHcwZ5",
        last_price: 4.13e-6,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 747.876204972505
      }
    }
  },
  %{
    asset: "BS1KFNR8zrXKBEWdUUvpaP6G57Hic3aESkwK7qQKdLpB",
    balance: 250.0,
    name: "Marquise Museum",
    priced_in: %{
      "WAVES" => %{
        asset: "BS1KFNR8zrXKBEWdUUvpaP6G57Hic3aESkwK7qQKdLpB",
        last_price: 1.75e-5,
        priced_in: "WAVES",
        volume: 19.90025929
      }
    }
  },
  %{
    asset: "HZk1mbfuJpmxU1Fs4AX5MWLVYtctsNcg6e2C6VKqK8zk",
    balance: 2.176e-5,
    name: "Litecoin",
    priced_in: %{
      "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu" => %{
        asset: "HZk1mbfuJpmxU1Fs4AX5MWLVYtctsNcg6e2C6VKqK8zk",
        last_price: 0.26700873,
        priced_in: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        volume: 89.5222443581639
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "HZk1mbfuJpmxU1Fs4AX5MWLVYtctsNcg6e2C6VKqK8zk",
        last_price: 0.0069835,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 3308.01951186921 
      },
      "WAVES" => %{
        asset: "HZk1mbfuJpmxU1Fs4AX5MWLVYtctsNcg6e2C6VKqK8zk",
        last_price: 52.32233717,
        priced_in: "WAVES",
        volume: 10353.2249989667
      }
    }
  },
  %{
    asset: "EYz8Zvs62D4d7F5ZgXHCWuzuFaZg63FYnfVQrTWQoLSK",
    balance: 5.4682,
    name: "Kolion",
    priced_in: %{ 
      "34N9YcEETLWn93qYQ64EsP1x89tSruJU44RrEMSXXEPJ" => %{
        asset: "EYz8Zvs62D4d7F5ZgXHCWuzuFaZg63FYnfVQrTWQoLSK",
        last_price: 0.644,
        priced_in: "34N9YcEETLWn93qYQ64EsP1x89tSruJU44RrEMSXXEPJ",
        volume: 250.114304919649
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "EYz8Zvs62D4d7F5ZgXHCWuzuFaZg63FYnfVQrTWQoLSK",
        last_price: 7.25e-5,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 2926.16778666486
      },
      "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p" => %{
        asset: "EYz8Zvs62D4d7F5ZgXHCWuzuFaZg63FYnfVQrTWQoLSK",
        last_price: 0.644,
        priced_in: "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
        volume: 307.242085049641
      },
      "WAVES" => %{
        asset: "EYz8Zvs62D4d7F5ZgXHCWuzuFaZg63FYnfVQrTWQoLSK",
        last_price: 0.545,
        priced_in: "WAVES",
        volume: 2886.36869668873 
      }
    }
  },
  %{
    asset: "62LyMjcr2DtiyF5yVXFhoQ2q414VPPJXjsNYp72SuDCH",
    balance: 1.6444e-4,
    name: "Bitcoin SV",
    priced_in: %{
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "62LyMjcr2DtiyF5yVXFhoQ2q414VPPJXjsNYp72SuDCH",
        last_price: 0.02714628,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 206.093558650586
      }
    }
  },
  %{
    asset: "8Yw4QmskrQauQeNjgh2fTQ4swmkNm85GTQzdHEf6QdUU",
    balance: 21.08167447,
    name: "ABOT",
    priced_in: %{
      "WAVES" => %{
        asset: "8Yw4QmskrQauQeNjgh2fTQ4swmkNm85GTQzdHEf6QdUU", 
        last_price: 0.00145051,
        priced_in: "WAVES",
        volume: 634.754564700846
      }
    }
  },
  %{
    asset: "43SjibPyAtSCoLHWD1hYi6hyZFcWFe95efPkpUJzpHyT",
    balance: 0.007806,
    name: "ZORIX",
    priced_in: %{
      "WAVES" => %{
        asset: "43SjibPyAtSCoLHWD1hYi6hyZFcWFe95efPkpUJzpHyT",
        last_price: 0.00154646,
        priced_in: "WAVES",
        volume: 13.0288009500292
      }
    }
  },
  %{
    asset: "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J",
    balance: 0.01,
    name: "WavesCommunity",
    priced_in: %{
      "4SvrfTcELe39nd28sa64mVEifH5DXpCFemZcu3yWkhqi" => %{
        asset: "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J",
        last_price: 8853.088,
        priced_in: "4SvrfTcELe39nd28sa64mVEifH5DXpCFemZcu3yWkhqi",
        volume: 831.353476003844
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J",
        last_price: 5.86e-6,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 10446.101397062
      },
      "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p" => %{
        asset: "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J",
        last_price: 0.05174493,
        priced_in: "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
        volume: 12307.581490077
      }, 
      "WAVES" => %{
        asset: "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J",
        last_price: 0.04330227,
        priced_in: "WAVES",
        volume: 13672.1244736743
      }
    }
  },
  %{
    asset: "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8",
    balance: 47.92501839,
    name: "Vostok",
    priced_in: %{
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8",
        last_price: 6.07e-6,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 3556.94601060728
      },
      "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p" => %{
        asset: "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8",
        last_price: 0.054104,
        priced_in: "DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
        volume: 1049.25219857421
      },
      "WAVES" => %{
        asset: "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8",
        last_price: 0.0455,
        priced_in: "WAVES",
        volume: 27887.4777878162
      }
    }
  },
  %{
    asset: "4uK8i4ThRGbehENwa6MxyLtxAjAo1Rj9fduborGExarC",
    balance: 0.01, 
    name: "MinersReward",
    priced_in: %{
      "4SvrfTcELe39nd28sa64mVEifH5DXpCFemZcu3yWkhqi" => %{
        asset: "4uK8i4ThRGbehENwa6MxyLtxAjAo1Rj9fduborGExarC",
        last_price: 2512.7785,
        priced_in: "4SvrfTcELe39nd28sa64mVEifH5DXpCFemZcu3yWkhqi",
        volume: 98.6542258569512
      }
    }
  },
  %{
    asset: "B3uGHFRpSUuGEDWjqB9LWWxafQj8VTvpMucEyoxzws5H",
    balance: 1.0e-8,
    name: "DASH",
    priced_in: %{
      "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu" => %{
        asset: "B3uGHFRpSUuGEDWjqB9LWWxafQj8VTvpMucEyoxzws5H",
        last_price: 0.39349802,
        priced_in: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        volume: 190.382097605763
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "B3uGHFRpSUuGEDWjqB9LWWxafQj8VTvpMucEyoxzws5H",
        last_price: 0.01025596,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 2673.85683197775
      },
      "WAVES" => %{
        asset: "B3uGHFRpSUuGEDWjqB9LWWxafQj8VTvpMucEyoxzws5H",
        last_price: 76.55451228,
        priced_in: "WAVES",
        volume: 3267.36280278265
      }
    }
  },
  %{
    asset: "bPWkA3MNyEr1TuDchWgdpqJZhGhfPXj7dJdr3qiW2kD",
    balance: 1.9e-7,
    name: "TurtleNetwork",
    priced_in: %{
      "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8" => %{
        asset: "bPWkA3MNyEr1TuDchWgdpqJZhGhfPXj7dJdr3qiW2kD",
        last_price: 0.10300031,
        priced_in: "4LHHvYGNKJUg5hj65aGD5vgScvCBmLpdRFtjokvCjSL8",
        volume: 1390.82650667832
      },
      "WAVES" => %{
        asset: "bPWkA3MNyEr1TuDchWgdpqJZhGhfPXj7dJdr3qiW2kD",
        last_price: 0.0035172,
        priced_in: "WAVES",
        volume: 14.0090539701
      }
    }
  },
  %{
    asset: "9TexWwLfw4wt4cvGaiYrFkiDHMZHsySVYehKZwehai4k",
    balance: 0.04166518,
    name: "BADA",
    priced_in: %{
      "gnoAVs6Fm8e9PsbdGV2thGJkCaJ6xnyLeu5Lp8o2JuU" => %{
        asset: "9TexWwLfw4wt4cvGaiYrFkiDHMZHsySVYehKZwehai4k",
        last_price: 100,
        priced_in: "gnoAVs6Fm8e9PsbdGV2thGJkCaJ6xnyLeu5Lp8o2JuU",
        volume: 35792.109663444
      }
    }
  },
  %{
    asset: "5WvPKSJXzVE2orvbkJ8wsQmmQKqTv9sGBPksV4adViw3",
    balance: 0.01668726,
    name: "Monero",
    priced_in: %{
      "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu" => %{
        asset: "5WvPKSJXzVE2orvbkJ8wsQmmQKqTv9sGBPksV4adViw3",
        last_price: 0.29742179,
        priced_in: "474jTeYx2r2Va35794tCScAXWJG9hU2HcgxzMowaZUnu",
        volume: 5969.95359099085
      },
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "5WvPKSJXzVE2orvbkJ8wsQmmQKqTv9sGBPksV4adViw3",
        last_price: 0.00775728,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 48550.086776975
      },
      "WAVES" => %{
        asset: "5WvPKSJXzVE2orvbkJ8wsQmmQKqTv9sGBPksV4adViw3",
        last_price: 57.92935315,
        priced_in: "WAVES",
        volume: 1738.83644844413
      }
    }
  },
  %{
    asset: "zMFqXuoyrn5w17PFurTqxB7GsS71fp9dfk6XFwxbPCy",
    balance: 4.7e-7,
    name: "Bitcoin Cash",
    priced_in: %{
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "zMFqXuoyrn5w17PFurTqxB7GsS71fp9dfk6XFwxbPCy",
        last_price: 0.03629774,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 184.873823322842
      },
      "WAVES" => %{
        asset: "zMFqXuoyrn5w17PFurTqxB7GsS71fp9dfk6XFwxbPCy",
        last_price: 275.31202697,
        priced_in: "WAVES",
        volume: 725.429712492121
      }
    }
  },
  %{
    asset: "5ZPuAVxAwYvptbCgSVKdTzeud9dhbZ7vvxHVnZUoxf4h",
    balance: 1.5561e-4,
    name: "ZrCoin",
    priced_in: %{
      "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS" => %{
        asset: "5ZPuAVxAwYvptbCgSVKdTzeud9dhbZ7vvxHVnZUoxf4h",
        last_price: 1.3999e-4,
        priced_in: "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS",
        volume: 258.498997176377
      },
      "WAVES" => %{
        asset: "5ZPuAVxAwYvptbCgSVKdTzeud9dhbZ7vvxHVnZUoxf4h",
        last_price: 1.07000071,
        priced_in: "WAVES",
        volume: 38.445487991766
      }
    }
  },
  %{
    asset: "HAeFQrVABF9UwYUJtCQLoYZ6yY7Z6752JNyT9nYKNdxL",
    balance: 90.0,
    name: "Eidolon.club",
    priced_in: %{
      "WAVES" => %{
        asset: "HAeFQrVABF9UwYUJtCQLoYZ6yY7Z6752JNyT9nYKNdxL",
        last_price: 4.99e-5,
        priced_in: "WAVES",
        volume: 312.9161673011
      }
    }
  }
]

Objective: Obtain priced_in values

Example: priced_in for 8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS is

["DG2xFkPdDwKUoBkzGAhQtLpSGzfXLiCYPEzeKH2Ad24p",
 "Ft8X1v1LTa1ABafufpaCWyVj8KkaxUWE6xBhW6sNFJck",
 "Gtb1WRznfchDnTh37ezoDTJ4wcoKaRsKqKjJjy7nm2zU"]

Code:


Enum.group_by(data, &(&1).asset) 
|> Map.get("8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS") 
|> Enum.flat_map(&(&1).priced_in) 
|> Enum.map(fn {keys, _} -> keys end)

Question:

What would be a more performant way to obtain the desired results? Thanks for your suggestions! :slight_smile:

1 Like
2

Does it need to be more efficient? If you really need to you can do it all with a single Enum.reduce (or a for reduce comprehension) if the order stays as currently is.

EDIT: What’s this from anyway? Is it a result of a SQL call? If so you could do it all in SQL far far more efficiently then. Or probably from GraphQL too.

1 Like
3

data is a map built from financial data pulled from a JSON api.

Since you mention SQL and my app will be requiring various Ecto tables, would it be more performant to just do everything via Ecto? Or is GraphQL a better way to go?

1 Like
4

Only if you are already putting it in the database, otherwise no.

Since you said it came from a JSON API then this is unrelated, ignore my mention of it. ^.^

1 Like
5

Preferably we’d (1)pull the JSON data, (2)parse it (3)transform it and then (4)push it to the db but by the time it’s ready to be recorded it’s already stale.

We could just write the raw data to some tables and then use Ecto to handle (2) and (3) but this seems even less performant.

What do you think? Is Ecto the way to go or do you have a better approach?

1 Like
6

Yeah no point to do that. ^.^;

I’d just transform it in code then. If you know the order will always be the same as the above then reducing it is fastest, otherwise a group_by is quite good. No real point in optimizing until the need actually arrives, keep it readable first. ^.^

1 Like
7

Do it in one run, recursively going through the enumerable and pattern matching. This way you could also return early once you found it if the asset id is unique.

How big is your input?

8 
iex(1)> Enum.reduce(data, &(&1).asset)

** (BadArityError) #Function<7.126501267/1 in :erl_eval.expr/5> with arity 1 called with 2 arguments (%{asset: "GAAZu8fsWfrZBFgweMMAFEcjrW9CFRxPEqmGcbSGWVDF", balance: 3.0, name: "K975", priced_in: %{"WAVES" => %{asset: "GAAZu8fsWfrZBFgweMMAFEcjrW9CFRxPEqmGcbSGWVDF", last_price: 3.9735e-4, priced_in: "WAVES", volume: 19.3305527100074}}}, %{asset: "51LxAtwBXapvvTFSbbh4nLyWFxH6x8ocfNvrXxbTChze", balance: 6.0e-6, name: "TORCorp", priced_in: %{"WAVES" => %{asset: "51LxAtwBXapvvTFSbbh4nLyWFxH6x8ocfNvrXxbTChze", last_price: 3.65e-4, priced_in: "WAVES", volume: 1998.66525750942}}})
    (elixir 1.10.1) lib/enum.ex:2111: Enum."-reduce/2-lists^foldl/2-0-"/3

How did I go wrong? (Still learning how to decode error messages.)

9

See: https://api.wavesplatform.com/v0/pairs

10

Reduce takes a function that takes two arguments, one is the element being processed and the second is the accumulator. ^.^

2 Likes