Need help and suggestions on metric dashboard

Questions / Help
, ,
1

i need help on implementing metric-dashboard for distribution metrics,
checked live_dashboard but live_dashboard not supporting distribution at the movement

i’m a new joinee to my team and fairly new to Elixir
i tasked to develop distribution metric-dashboard

data coming from TelemetryMetricsPrometheus.Core at /metrics endpoint
i’m thinking to parse metrics data into maps or list and use some UI library to represent

  1. what is the best UI library to represent distribution metrics?
  2. what type of data structure these libraries need? if i wanted to parse data into some structure(maps, list, keyword-list)
  3. are there any examples to take references?

currently i’m trying this to parse data its half baked code though :slight_smile:
i’m doing this on livebbok

defmodule P do

  def data_within_curl(labels) do
    labels
    |> remove_string(~r/"/)
    |> String.split(",")
    |> Enum.reduce(%{}, fn match, acc ->
      [key, val] = String.split(match, "=")
      Map.put(acc, key, val)
    end)
  end

  def remove_string(str, regex, replacement \\ "") do
    Regex.replace(regex, str, replacement)
  end

  def check_for_help?(content), do: String.match?(content, ~r/HELP/)
  def check_for_type?(content), do: String.match?(content, ~r/TYPE/)
  def check_for_curly?(content), do: String.match?(content, ~r/{|}/)
  def check_for_no_curly?(content), do: !String.match?(content, ~r/{|}/)
end

"""
# HELP app_name_self_repair_duration

# TYPE app_name_self_repair_duration gauge

app_name_self_repair_duration 0.15916584

# HELP app_name_db_duration

# TYPE app_name_db_duration histogram

app_name_db_duration_bucket{query="get_transaction",le="0.01"} 179807

app_name_db_duration_bucket{query="get_transaction",le="0.025"} 180397

app_name_db_duration_bucket{query="get_transaction",le="0.05"} 183557

app_name_db_duration_bucket{query="get_transaction",le="0.1"} 188560

app_name_db_duration_bucket{query="get_transaction",le="0.3"} 188692

app_name_db_duration_bucket{query="get_transaction",le="0.5"} 188695

app_name_db_duration_bucket{query="get_transaction",le="0.8"} 188695

app_name_db_duration_bucket{query="get_transaction",le="1"} 188695

app_name_db_duration_bucket{query="get_transaction",le="1.5"} 188695

app_name_db_duration_bucket{query="get_transaction",le="+Inf"} 188695

app_name_db_duration_sum{query="get_transaction"} 555.663747302

app_name_db_duration_count{query="get_transaction"} 188695
"""
|> String.split("\n")
|> Enum.reduce(%{}, fn line, acc ->
  IO.puts(line)
  IO.inspect(acc, label: "acc-map")

  cond do
    P.check_for_help?(line) ->
      acc

    P.check_for_type?(line) ->
      [_, _, metric_name, type] = Regex.run(~r/(.*\s)(.*\s)(.*)/, line)
      trim_trailing = String.trim_trailing(metric_name)
      new_metric = Enum.join([trim_trailing, type], "_")
      IO.puts(new_metric)
      Map.update(acc, new_metric, [], fn val -> IO.puts(val) end)

    P.check_for_curly?(line) ->
      [_, metric_name, labels, value] = Regex.run(~r/(.*){(.*)}(.*)/, line)

      labels_data = P.data_within_curl(labels)
      IO.inspect(labels_data, label: "labels_data")

      bucket_map = Map.merge(labels_data, %{"value" => value})
      IO.inspect(bucket_map, label: "with curly")
      Map.put_new(acc, metric_name, bucket_map)

    P.check_for_no_curly?(line) ->
      [_, metric_name, value] = Regex.run(~r/(.*\s)(.*)/, line)
      Map.put_new(acc, metric_name, %{"value" => value})
      IO.inspect(value, label: "no curly")
  end
end)

any help or suggestions are greatly appreciated
thank you

2

i came up with suitable solution after doing some research
parse the data and convert into JSON structure then use some charting libraries(EChart, Vega-lite) to present the data.

JSON structure:

[
  {
    "help": "app_name_db_duration",
    "type": "histogram",
    "metrics": [
      {
        "labels": {
          "method": "query",
          "handler": "get_transaction"
        },
        "buckets": {
          "0.01": "443481",
          "0.025": "447919",
          "0.05": "456449",
          "0.1": "472222"
        },
        "count": "472624",
        "sum": "1894.9452645920032"
      },
      {
        "labels": {
          "method": "query",
          "handler": "get_transaction_chain"
        },
        "buckets": {
          "0.99": "3542.9",
          "0.9": "1202.3",
          "0.5": "1002.8"
        },
        "count": "4",
        "sum": "345.01"
      }
    ]
  },
  {
    "help": "app_name_replication_validation_duration",
    "type": "histogram",
    "metrics": [
      {
        "buckets": {
          "0.99": "3542.9",
          "0.9": "1202.3",
          "0.5": "1002.8"
        },
        "count": "4",
        "sum": "345.01"
      }
    ]
  },
]

so far i have coded this functions:

defmodule K do
  def parse_help(content) do
    [_, _, name] = Regex.run(~r/(.*\s)(.*)/, content)
    %{"help" => name}
  end

  def parse_type(content) do
    [_, _, _, type] = Regex.run(~r/(.*\s)(.*\s)(.*)/, content)
    %{"type" => type}
  end

  def parse_with_curly(content) do
    [_, _metric_name, lab, value] = Regex.run(~r/(.*){(.*)}(.*)/, content)
    [bucket, label, handler] = data_within_curl(lab)
    bucket_val = %{bucket => value |> remove_string(~r/ /, "")}
    buckets =%{"buckets" => bucket_val}
    labels = %{"labels" => %{"method" => label, "handler" => handler}}
    {:ok, buckets, labels}
    
  end

  def parse_with_curly_no_label(content) do
    [_, _metric_name, lab, value] = Regex.run(~r/(.*){(.*)}(.*)/, content)
    [bucket] = data_within_curl(lab)
    bucket_val = %{bucket => value |> remove_string(~r/ /, "")}
    %{"buckets" => bucket_val}
  end

  def parse_sum_count_with_label(content) do
    [_, name, _lab, value] = Regex.run(~r/_(sum|count){(.*)}(.*)/, content)
    %{name => value |> remove_string(~r/ /, "")}
  end

  def parse_sum_count_no_label_no_curly(content) do
    [_, name, value] = Regex.run(~r/_(sum|count)(.*)/, content)
    %{name => value |> remove_string(~r/ /, "")}
  end

  defp data_within_curl(labels) do
    labels
    |> remove_string(~r/"/)
    |> String.split(",")
    |> Enum.reduce([], fn match, acc ->
      [key, val] = String.split(match, "=")
      [key | [val | acc]] |> Enum.reject(fn le -> le == "le" end)
    end)
  end

  def remove_string(str, regex, replacement \\ "") do
    Regex.replace(regex, str, replacement)
  end

  def check_for_help?(content), do: String.match?(content, ~r/HELP/)
  def check_for_type?(content), do: String.match?(content, ~r/TYPE/)
  def check_for_curly?(content), do: String.match?(content, ~r/{|}/)
  def check_for_no_curly?(content), do: !String.match?(content, ~r/{|}|(HELP)|(TYPE)/)
  def check_for_sum_count_with_curly?(content), do: String.match?(content, ~r/_sum{|_count{/)
  def check_for_le_with_curly?(content), do: String.match?(content, ~r/{le/)
end

i’m trying these functions:

"""
# HELP app_name_db_duration

# TYPE app_name_db_duration histogram

app_name_db_duration_bucket{query="get_transaction",le="0.01"} 179807

app_name_db_duration_bucket{query="get_transaction",le="0.025"} 180397

app_name_db_duration_bucket{query="get_transaction",le="0.05"} 183557

app_name_db_duration_bucket{query="get_transaction",le="0.1"} 188560

app_name_db_duration_bucket{query="get_transaction",le="0.3"} 188692
"""
|> String.split("\n")
|> Enum.reduce([], fn line, acc ->
  IO.puts(line)
  IO.inspect(acc, label: "acc-map")
  IO.inspect(length(acc), label: "acc-size")

  cond do
    K.check_for_help?(line) ->
      help = K.parse_help(line)
      [help | acc]
      
    K.check_for_type?(line) ->
      [K.parse_type(line) | acc]

    K.check_for_curly?(line) ->
      cond do
        K.check_for_sum_count_with_curly?(line) ->
          sum_or_count = K.parse_sum_count_with_label(line)
          [sum_or_count | acc]
        K.check_for_le_with_curly?(line) ->
          le = K.parse_with_curly_no_label(line)
          [le | acc]
        K.check_for_curly?(line) ->
          {:ok, buckets, labels} = K.parse_with_curly(line)
          metrics = %{"metrics" => [{labels, buckets}]}
          [metrics | acc]
      end
      
    K.check_for_no_curly?(line) ->
      sum = K.parse_sum_count_no_label_no_curly(line)
      [sum | acc]
  end
end)

this is the output:

  # HELP app_name_db_duration
acc-map: []
acc-size: 0
# TYPE app_name_db_duration histogram
acc-map: [%{"help" => "app_name_db_duration"}]
acc-size: 1
app_name_db_duration_bucket{query="get_transaction",le="0.01"} 443481
acc-map: [%{"type" => "histogram"}, %{"help" => "app_name_db_duration"}]
acc-size: 2
app_name_db_duration_bucket{query="get_transaction",le="0.025"} 447919
acc-map: [
  %{
    "metrics" => [
      {%{"labels" => %{"handler" => "get_transaction", "method" => "query"}},
       %{"buckets" => %{"0.01" => "443481"}}}
    ]
  },
  %{"type" => "histogram"},
  %{"help" => "app_name_db_duration"}
]
acc-size: 3
app_name_db_duration_bucket{query="get_transaction",le="0.05"} 456449
acc-map: [
  %{
    "metrics" => [
      {%{"labels" => %{"handler" => "get_transaction", "method" => "query"}},
       %{"buckets" => %{"0.025" => "447919"}}}
    ]
},
  %{
    "metrics" => [
      {%{"labels" => %{"handler" => "get_transaction", "method" => "query"}},
       %{"buckets" => %{"0.01" => "443481"}}}
    ]
  },
  %{"type" => "histogram"},
  %{"help" => "app_name_db_duration"}
]

but the problem now is the label names and metrics names are repeating in every map

how can i tell to my functions, label names and metrics name not to take more than once
how can i take bucket numbers every time? and ignore labels