Found a really simple way to solve this. Just read the data-header, then parse row by row:
def read(src, cnt \\ nil) do
{:ok, fid} = :file.open("_cache/#{src}", [:raw, :read, :binary])
{:h, start, n} = _p(fid, :h)
cnt = cnt || n
Logger.debug("Reading: #{cnt} of #{n} rows")
Process.put(:start_pos, start)
for i <- 1..cnt do
{:r, start, row} = _p(fid, Process.get(:start_pos), :r)
Process.put(:start_pos, start)
Logger.debug(inspect({i, row}, @format))
end
:file.close(fid)
end
# READER
defp _p(fid, :h) do
# 000007775B4C4900000136490000000049FFFFFFFF4C490000006749000000004900000005
{:ok, <<n::32>>} = :file.pread(fid, 33, 4)
{:h, 37, n}
end
defp _p(fid, start, :r) do
# 490000000049000000006A62 00000165 3136333737303....
{:ok, <<l::32>>} = :file.pread(fid, start + 12, 4)
{:ok, b} = :file.pread(fid, start + 16, l)
{:r, start + l + 18, _split(b)}
end
defp _split(v) do
b = for(<<c <- v>>, c in 32..126 || c in [252, 253, 254], into: "", do: <<c>>)
b = :binary.replace(b, <<253>>, ";", [:global])
b = :binary.replace(b, <<252>>, "^", [:global])
:binary.split(b, <<254>>, [:global])
end
