I love this problem. I tried several things that I just knew you be faster, only to learn the hard way that they were not.
Here’s the faster thing I was able to come up with:
defmodule WordCounter do
def run(table, pattern, parent) do
spawn(__MODULE__, :read_and_count, [table, pattern, parent])
end
def read_and_count(table, pattern, parent) do
case IO.binread(:line) do
:eof ->
send(parent, {:done, self()})
line ->
line
|> String.split(pattern)
|> Enum.each(fn word ->
:ets.update_counter(table, word, {2, 1}, {word, 0})
end)
read_and_count(table, pattern, parent)
end
end
def wait_on(pid) do
receive do
{:done, ^pid} ->
:ok
end
end
end
table = :ets.new(:words, [:public, write_concurrency: true])
pattern = :binary.compile_pattern([" ", "\n"])
Stream.repeatedly(fn -> WordCounter.run(table, pattern, self()) end)
|> Enum.take(System.schedulers_online)
|> Enum.each(fn pid -> WordCounter.wait_on(pid) end)
:ets.tab2list(table)
|> Enum.sort(fn {_, a}, {_, b} -> b < a end)
|> Enum.map(fn {word, count} ->
[String.pad_leading(Integer.to_string(count), 8), " ", word, "\n"]
end)
|> IO.binwrite
I did like that I was able to get some decent speed with a still pretty straight forward approach. (It just reads lines in multiple processes.)
