I wish you all a happy and healthy new year!
I have configured the pool size as advised to 20 and the max_concurrency to 18.
This seems to have no effect as I still have the same issue as before.
config/config.exs
import Config
config :file_watch_example, :ecto_repos, [Store]
config :file_watch_example, Store,
database: "store",
username: "",
password: "",
hostname: "db",
pool_size: 20,
migration_timestamps: [type: :utc_datetime_usec],
migration_lock: nil,
queue_target: 10_000
config :logger, :console,
level: :debug,
format: "[$level] $message $metadata\n",
metadata: [:error_code, :file]
config :logger,
backends: [{LoggerFileBackend, :debug}]
config :logger, :debug,
path: "/var/log/elixir_app/debug.log",
level: :debug
lib/ingest_csv.ex
defmodule IngestCSV do
alias NimbleCSV.RFC4180, as: YourCSV
require Logger
NimbleCSV.define(YourCSV, separator: ";")
def load(path) do
path
|> Path.expand()
|> File.stream!(read_ahead: 524_288)
|> YourCSV.parse_stream()
|> Stream.chunk_every(500)
|> Task.async_stream(fn rows ->
Enum.map(rows, fn [name, bar_code, price, currency] ->
FileWatchExample.Product.create_product(%{name: name, bar_code: bar_code, price: price, currency: currency})
end)
end, max_concurrency: 18, on_timeout: :kill_task, ordered: false)
|> Stream.run()
end
end
grep -i error /var/log/elixir_app/debug.log | tail -3
07:43:22.538 [info] MyXQL.Connection (#PID<0.230.0>) disconnected: ** (DBConnection.ConnectionError) client #PID<0.5886.0> exited
07:43:22.541 [info] MyXQL.Connection (#PID<0.226.0>) disconnected: ** (DBConnection.ConnectionError) client #PID<0.5889.0> exited
07:43:22.542 [info] MyXQL.Connection (#PID<0.240.0>) disconnected: ** (DBConnection.ConnectionError) client #PID<0.5890.0> exited
The MyXQL.Connection Error is the only one I am facing currently:
grep -i error /var/log/elixir_app/debug.log | grep -v MyXQL.Connection | wc -l
0
MySQL “show variables:” :
concurrent_insert | AUTO
connect_timeout | 10
max_connect_errors | 100
max_connections | 1000
max_insert_delayed_threads | 20
max_user_connections | 0
The Elixir-App and MySQL run in Docker and no other containers are running when doing my tests.
