I am trying to make a script which tends to do a few steps.
- get 2 date inputs
- format those dates and get pair of dates with a starting date and an ending one.
Just for the sake of convenience, I am putting the full script, you can place it any file and it will run if you want to test it.
defmodule Extractions do
@format ~r[(?<start_hour>\d{2}):(?<start_minute>\d{2})-(?<end_hour>\d{2}):(?<end_minute>\d{2})]
def start do
start_date = Calendar.DateTime.from_erl!({{2020, 1, 25},{0, 29, 10}}, "Etc/UTC", {123456, 6}) #|> Calendar.DateTime.shift_zone!("Europe/Dublin")
end_date = Calendar.DateTime.from_erl!({{2020, 2, 2},{0, 29, 10}}, "Etc/UTC", {123456, 6}) #|> Calendar.DateTime.shift_zone!("Europe/Dublin")
schedule = %{
"Friday" => ["08:00-18:00"],
"Monday" => ["08:00-18:00"],
"Saturday" => [],
"Sunday" => [],
"Thursday" => ["08:00-18:00"],
"Tuesday" => ["08:00-18:00"],
"Wednesday" => ["08:00-18:00"]
}
|> Enum.filter(fn {_, v} -> length(v) != 0 end)
|> Enum.into(%{})
days =
schedule
|> Enum.map(fn(sc) ->
{day, hours} = sc
if length(hours) != 0, do: day
end) |> Enum.filter(& !is_nil(&1))
camera_exid = "waxie-jolxd"
interval = 1200
all_days =
Calendar.Date.days_after_until(start_date, end_date, true)
|> Enum.filter(fn(day) ->
Enum.member?(days, day |> Calendar.Strftime.strftime!("%A"))
end)
valid_dates =
all_days
|> get_date_pairs(camera_exid, schedule)
{get_expected_count} =
Enum.reduce(valid_dates, {0}, fn date_pair, {count} ->
%{starting: starting, ending: ending} = date_pair
{:ok, after_seconds, 0, :after} = Calendar.DateTime.diff(ending, starting)
{count + (after_seconds / interval)}
end)
valid_dates
end
defp get_date_pairs(dates, camera_exid, schedule) do
dates
|> Enum.map(fn date ->
schedule[Calendar.Strftime.strftime!(date, "%A")]
|> get_head_tail
|> Enum.map(fn timings -> Regex.named_captures(@format, timings |> List.first) end)
|> Enum.map(fn schedule_time ->
Map.merge(
%{
"year" => strft_date(date, "%Y"),
"month" => strft_date(date, "%m"),
"day" => strft_date(date, "%d")
},
schedule_time
)
end)
end)
|> List.flatten
|> Enum.map(fn date_tuple ->
{starting, ending} = parse_schedule_times(date_tuple)
%{
starting: Calendar.DateTime.from_erl!(starting, "Etc/UTC", {123456, 6}) |> shift_zone,
ending: Calendar.DateTime.from_erl!(ending, "Etc/UTC", {123456, 6}) |> shift_zone
}
end)
end
defp strft_date(date, pattern), do: Calendar.Strftime.strftime!(date, pattern)
defp shift_zone(date, timezone \\ "Europe/Dublin") do
date |> Calendar.DateTime.shift_zone!(timezone)
end
defp parse_schedule_times(%{"end_hour" => end_hour, "end_minute" => end_minute, "start_hour" => start_hour, "start_minute" => start_minute, "year" => year, "month" => month, "day" => day}) do
{{{String.to_integer(year), String.to_integer(month), String.to_integer(day)}, {String.to_integer(start_hour), String.to_integer(start_minute), 0}}, {{String.to_integer(year), String.to_integer(month), String.to_integer(day)}, {String.to_integer(end_hour), String.to_integer(end_minute), 0}}}
end
def get_head_tail([]), do: []
def get_head_tail(nil), do: []
def get_head_tail([head|tail]) do
[[head]|get_head_tail(tail)]
end
end
now when it get starts with above those dates.
start_date = Calendar.DateTime.from_erl!({{2020, 1, 25},{0, 29, 10}}, "Etc/UTC", {123456, 6})
end_date = Calendar.DateTime.from_erl!({{2020, 2, 2},{0, 29, 10}}, "Etc/UTC", {123456, 6})
the end result will be
iex(4)> Extractions.start
[
%{
ending: #DateTime<2020-01-27 18:00:00.123456+00:00 GMT Europe/Dublin>,
starting: #DateTime<2020-01-27 08:00:00.123456+00:00 GMT Europe/Dublin>
},
%{
ending: #DateTime<2020-01-28 18:00:00.123456+00:00 GMT Europe/Dublin>,
starting: #DateTime<2020-01-28 08:00:00.123456+00:00 GMT Europe/Dublin>
},
%{
ending: #DateTime<2020-01-29 18:00:00.123456+00:00 GMT Europe/Dublin>,
starting: #DateTime<2020-01-29 08:00:00.123456+00:00 GMT Europe/Dublin>
},
%{
ending: #DateTime<2020-01-30 18:00:00.123456+00:00 GMT Europe/Dublin>,
starting: #DateTime<2020-01-30 08:00:00.123456+00:00 GMT Europe/Dublin>
},
%{
ending: #DateTime<2020-01-31 18:00:00.123456+00:00 GMT Europe/Dublin>,
starting: #DateTime<2020-01-31 08:00:00.123456+00:00 GMT Europe/Dublin>
}
]
now. What I am trying to do is: I need to pass each date pair to a process, in which.
Process will start from starting date and increase it with interval as seconds and reach upto ending date.
During this increase…
- It will download files from seaweedfs(A file system REST API.) for the date.
- Save it and put in dropbox batch
- Push those images to dropbox on 1000 no of batches or less if its already on ending date.
Now My question is: I want to set my scenarios in genStage or in Flow. so that with the above Pair. of dates.
How I can use GenStage or Flow to make the process faster.
PS: There can be any date range. let suppose a year. and also there can be more than 10000 pairs of dates.
each date pair, can be working our also can be a full day.
Any guidance would be perfect, I am learning GenStage, I am getting nothing from counter loop examples. any guidline will do. thanks