I tried this (building off of the previous performant solution):
index_file
|> stream_file()
|> Task.async_stream(
fn line ->
path = :lists.droplast(line)
{:ok, contents} = :prim_file.read_file(path)
{:ok, %{"paths" => txt_paths}} = Jason.decode(contents)
txt_paths
[]
end,
max_concurrency: 10,
ordered: false
)
|> Stream.flat_map(fn {:ok, results} -> results end)
|> Task.async_stream(
fn path ->
match?({:ok, _}, :prim_file.read_file_info(path))
end,
max_concurrency: 10,
ordered: false
)
|> Stream.run()
I streamed the file with this code (I think I’m probably reinventing wheels here, but it was educational):
def stream_file(input_file) do
Stream.resource(
fn ->
{:ok, file} = :file.open(input_file, [:raw, :read, read_ahead: 8192])
file
end,
fn file ->
case :file.read_line(file) do
{:ok, line} ->
{[line], file}
:eof ->
{:halt, file}
end
end,
fn file -> :file.close(file) end
)
end
This performed more or less the same as the other solutions.
