Issue: Response count mismatch in decode function under concurrent load
Setup: 4-node cluster, 500 concurrent HTTP requests dispatched via Task.async. All requests return within 2ms max latency.
Problem: Instrumented the decode function with entry/exit log markers. Entry log shows all 500 invocations as expected, but exit log shows only ~179 completions — roughly 64% of decode calls are not reaching the exit log point.
I am using Task.yield_many(spec.kill_cutoff_time_ms) with 4.5 seconds as timeout here. Each task will make http request and decode call Jason.decode
I have tried with Nimble ParserPool but I could reach only 232 at maximum on 64 Core system with pool size of
200
Here is the ParserPool.
defmodule GrncHub.ParserPool do
@behaviour NimblePool
require Logger
@pool_size 500
def child_spec(opts) do
%{
id: **MODULE**,
start: {**MODULE**, :start_link, \[opts\]},
type: :worker,
restart: :permanent,
shutdown: 500
}
end
def start_link(\_) do
NimblePool.start_link(worker: {**MODULE**, nil}, pool_size: @pool_size, name: **MODULE**)
end
@impl true
def init_worker(pool_state), do: {:ok, nil, pool_state}
@impl true
def handle_checkout(:parse, \_from, worker_state, pool_state) do
{:ok, worker_state, worker_state, pool_state}
end
@impl true
def terminate_worker(\_reason, _worker_state, pool_state) do
{:ok, pool_state}
end
@doc """
Returns {:ok, decoded} | {:error, :invalid_json} | {:error, :worker_crashed} | {:error, :invalid_input}
"""
def parse(payload, content_type, parser) when is_binary(payload) do
parser = default_parser(content_type, parser)
try do
NimblePool.checkout!(
__MODULE__,
:parse,
fn _from, worker_state ->
{GrncHub.Parser.decode(payload, content_type, parser), worker_state}
end,
:infinity
)
catch
:exit, reason ->
Logger.error("ParsePool worker crashed: #{inspect(reason)}")
{:error, %{"error" => "worker_crashed"}}
end
end
def parse(\_invalid), do: {:error, %{"error" => "worker_crashed"}}
defp default_parser(:json, nil), do: Jason
defp default_parser(\_content_type, parser), do: parser
end
