It shouldn’t be, heres a modified version of my processing with the logic parts (maps & reduce) removed.
temp_file
|> File.stream!([], 1024 * 1024 * 10)
|> StreamGzip.gunzip()
|> Stream.concat([:end])
|> Stream.transform("", fn
:end, prev ->
{[prev], ""}
chunk, prev ->
[last_line | lines] =
String.split(prev <> chunk, "\n")
|> Enum.reverse()
{Enum.reverse(lines), last_line}
end)
|> Flow.from_enumerable(stages: 50)
|> Flow.map(&parse_record/1)
|> Flow.partition(stages: 1)
|> Flow.reduce(fn -> %{} end, fn record, map ->
# aggregate into map
end)
