I wanted to play with streams too, so I had a look inside File.Stream and hacked together this very incomplete bit of code:
defmodule TokenStream do
defstruct path: nil, token_mod: nil
def new(path, token_mod) do
%TokenStream{path: path, token_mod: token_mod}
end
defimpl Enumerable do
def reduce(%{path: path, token_mod: token_mod}, acc, fun) do
start_fun = fn ->
case :file.open(path, :read) do
{:ok, file} ->
{file, 1}
{:error, reason} ->
raise File.Error, reason: reason, action: "tokenstream", path: path
end
end
next_fun = fn {file, line} ->
case :io.request(file, {:get_until, '', token_mod, :tokens, [line]}) do
{:eof, line} ->
{:halt, {file, line}}
{:error, reason} ->
raise IO.StreamError, reason: reason
{:ok, tokens, line} ->
{[tokens], {file, line}}
end
end
after_fun = fn {file, _} ->
:file.close(file)
end
Stream.resource(start_fun, next_fun, after_fun).(acc, fun)
end
end
end
This works because you have a terminator character which plays nice with io:request according to the leex docs. So in my chess_parser.xrl I changed the * line to return an end_token rather than a token:
\* : {end_token, {finish}}.
Then this works in the console:
iex(19)> TokenStream.new("test/file.txt", :chess_parser) |> Stream.map(& &1) |> Enum.to_list()
[
[
{:start, 1, 'A02'},
{:description, 1, '"Bird: From Gambit, Langheld Gambit"'},
{:move_number, 1, 1},
{:move, 1, 'f4'},
{:move, 1, 'e5'},
{:move_number, 1, 2},
{:move, 1, 'fxe5'},
{:move, 1, 'd6'},
{:move_number, 1, 3},
{:move, 1, 'exd6'},
{:move, 1, 'Nf6'},
{:finish}
],
[
{:start, 2, 'A02'},
{:description, 2, '"Bird: From Gambit, 3...Bxd6"'},
{:move_number, 2, 1},
{:move, 2, 'f4'},
{:move, 2, 'e5'},
{:move_number, 2, 2},
{:move, 2, 'fxe5'},
{:move, 2, 'd6'},
{:move_number, 2, 3},
{:move, 2, 'exd6'},
{:move, 2, 'Bxd6'},
{:finish}
],
[
{:start, 3, 'A02'},
{:description, 3, '"Bird: From Gambit, Lipke"'},
{:move_number, 3, 1},
{:move, 3, 'f4'},
{:move, 3, 'e5'},
{:move_number, 3, 2},
{:move, 3, 'fxe5'},
{:move, 3, 'd6'},
{:move_number, 3, 3},
{:move, 3, 'exd6'},
{:move, 3, 'Bxd6'},
{:move_number, 3, 4},
{:move, 3, 'Nf3'},
{:move, 3, 'Nh6'},
{:move_number, 3, 5},
{:move, 3, 'd4'},
{:finish}
],
[
{:start, 4, 'A02'},
{:description, 4, '"Bird: From Gambit, Lasker Variation"'},
{:move_number, 5, 1},
{:move, 5, 'f4'},
{:move, 5, 'e5'},
{:move_number, 5, 2},
{:move, 5, 'fxe5'},
{:move, 5, 'd6'},
{:move_number, 5, 3},
{:move, 5, 'exd6'},
{:move, 5, 'Bxd6'},
{:move_number, 5, 4},
{:move, 5, 'Nf3'},
{:move, 5, 'g5'},
{:finish}
],
[
{:start, 6, 'A02'},
{:description, 6, '"Bird: From Gambit, Lasker, 5.d4"'},
{:move_number, 7, 1},
{:move, 7, 'f4'},
{:move, 7, 'e5'},
{:move_number, 7, 2},
{:move, 7, 'fxe5'},
{:move, 7, 'd6'},
{:move_number, 7, 3},
{:move, 7, 'exd6'},
{:move, 7, 'Bxd6'},
{:move_number, 7, 4},
{:move, 7, 'Nf3'},
{:move, 7, 'g5'},
{:move_number, 7, 5},
{:move, 7, 'd4'},
{:finish}
]
]
