{"schema_version":"1.0","gateway":{"key":"litellm-python","display":"LiteLLM · Python","lang":"Python","cls":"LLM gateway","repo":"https://github.com/BerriAI/litellm","version":"v1.108.3","stars":63241,"stars_as_of":"2026-11-14","first_commit":"2023-07-27","matrix":{"gateway":"litellm-python","build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","matrix_version":null,"served":true,"serve_error":"","upstream_shape":"","upstream_note":"","egress_configured":"","probe_first":false,"capability_note":"","cell_perf_sweep":true,"sweep_rung_selection":null,"sweep_ttft_ms":null,"p99_ceiling_ms":null,"sweep_dur":null,"cell_stream":true,"cell_memory":true,"memory":null,"cells":{},"upstreams":{"cohere":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"cohere":{"served":true,"reason":null,"status":"","path":"/cohere/v2/chat","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":11850,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":12623,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":12651,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":12063.492063492064,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":82.89473684210526,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.24961199687176264,"certified":true,"suppressed":false},"cost_window_ok":{"value":504,"certified":true,"suppressed":false},"cost_window_rps":{"value":83,"certified":true,"suppressed":false},"cost_threads":{"value":6,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.051587301587301584,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":83,"certified":true,"suppressed":false},"concurrency":1,"p99_us":12764,"first_disqualified_conc":4,"lower_bound":false,"samples":499},{"bound_ms":100,"rps":{"value":83,"certified":true,"suppressed":false},"concurrency":1,"p99_us":12764,"first_disqualified_conc":8,"lower_bound":false,"samples":499},{"bound_ms":null,"rps":{"value":112,"certified":true,"suppressed":false},"concurrency":1024,"p99_us":3940819,"first_disqualified_conc":8192,"lower_bound":false,"samples":674}],"sweep":[{"conc":1,"ok":499,"rps":83,"p99_us":12710,"fail":0},{"conc":1,"ok":500,"rps":83,"p99_us":12722,"fail":0},{"conc":1,"ok":499,"rps":83,"p99_us":12764,"fail":0},{"conc":2,"ok":499,"rps":83,"p99_us":25278,"fail":0},{"conc":2,"ok":499,"rps":83,"p99_us":25286,"fail":0},{"conc":2,"ok":502,"rps":83,"p99_us":25191,"fail":0},{"conc":4,"ok":500,"rps":83,"p99_us":55368,"fail":0},{"conc":4,"ok":462,"rps":76,"p99_us":73697,"fail":0},{"conc":4,"ok":500,"rps":83,"p99_us":55645,"fail":0},{"conc":8,"ok":502,"rps":83,"p99_us":112408,"fail":0},{"conc":8,"ok":499,"rps":83,"p99_us":112782,"fail":0},{"conc":8,"ok":501,"rps":83,"p99_us":112620,"fail":0},{"conc":16,"ok":503,"rps":83,"p99_us":267160,"fail":0},{"conc":16,"ok":499,"rps":83,"p99_us":253671,"fail":0},{"conc":16,"ok":504,"rps":83,"p99_us":222623,"fail":0},{"conc":32,"ok":502,"rps":83,"p99_us":702942,"fail":0},{"conc":32,"ok":499,"rps":83,"p99_us":776613,"fail":0},{"conc":32,"ok":507,"rps":84,"p99_us":677806,"fail":0},{"conc":64,"ok":498,"rps":82,"p99_us":3040076,"fail":0},{"conc":64,"ok":494,"rps":82,"p99_us":3056278,"fail":0},{"conc":64,"ok":497,"rps":82,"p99_us":2931697,"fail":0},{"conc":128,"ok":490,"rps":81,"p99_us":5447807,"fail":0},{"conc":128,"ok":507,"rps":84,"p99_us":5461208,"fail":0},{"conc":128,"ok":513,"rps":85,"p99_us":1650814,"fail":0},{"conc":256,"ok":483,"rps":80,"p99_us":5324722,"fail":0},{"conc":256,"ok":454,"rps":75,"p99_us":5265943,"fail":0},{"conc":256,"ok":498,"rps":82,"p99_us":5405204,"fail":0},{"conc":512,"ok":481,"rps":80,"p99_us":3975195,"fail":0},{"conc":512,"ok":620,"rps":103,"p99_us":3655729,"fail":0},{"conc":512,"ok":511,"rps":85,"p99_us":5348350,"fail":0},{"conc":1024,"ok":674,"rps":112,"p99_us":3940819,"fail":0},{"conc":1024,"ok":653,"rps":107,"p99_us":3786357,"fail":0},{"conc":1024,"ok":469,"rps":78,"p99_us":5548585,"fail":0},{"conc":2048,"ok":675,"rps":112,"p99_us":3933167,"fail":0},{"conc":2048,"ok":240,"rps":39,"p99_us":5729416,"fail":0},{"conc":2048,"ok":512,"rps":85,"p99_us":5345907,"fail":0},{"conc":4096,"ok":395,"rps":65,"p99_us":5688739,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":403},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":1057}],"rungs":[{"conc":1,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":12722,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":25278,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":55645,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":112620,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":253671,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":702942,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":82,"certified":true,"suppressed":false},"p99_us":3040076,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":84,"certified":true,"suppressed":false},"p99_us":5447807,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":80,"certified":true,"suppressed":false},"p99_us":5324722,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":3975195,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":107,"certified":true,"suppressed":false},"p99_us":3940819,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5345907,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":65,"certified":true,"suppressed":false},"p99_us":5688739,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"cohere>cohere is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 503 successful gateway round trip(s) and 264219 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":503,"direct_c1_samples":264219},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.81640625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false},"peak_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":663.33203125,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":81,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.81640625},{"t_s":2,"rss_mib":641.578125},{"t_s":2,"rss_mib":641.765625},{"t_s":5,"rss_mib":643.1484375},{"t_s":6,"rss_mib":643.1484375},{"t_s":8,"rss_mib":643.95703125},{"t_s":9,"rss_mib":643.95703125},{"t_s":10,"rss_mib":644.765625},{"t_s":12,"rss_mib":644.765625},{"t_s":14,"rss_mib":645.06640625},{"t_s":15,"rss_mib":645.06640625},{"t_s":16,"rss_mib":645.92578125},{"t_s":18,"rss_mib":645.92578125},{"t_s":20,"rss_mib":646.1796875},{"t_s":21,"rss_mib":646.1796875},{"t_s":24,"rss_mib":646.64453125},{"t_s":24,"rss_mib":646.69921875},{"t_s":24,"rss_mib":647.04296875},{"t_s":27,"rss_mib":647.04296875},{"t_s":30,"rss_mib":647.68359375},{"t_s":30,"rss_mib":647.69140625},{"t_s":30,"rss_mib":647.95703125},{"t_s":33,"rss_mib":647.96484375},{"t_s":36,"rss_mib":649.38671875},{"t_s":36,"rss_mib":649.38671875},{"t_s":39,"rss_mib":649.38671875},{"t_s":41,"rss_mib":652.51953125},{"t_s":42,"rss_mib":652.51953125},{"t_s":44,"rss_mib":652.84765625},{"t_s":45,"rss_mib":652.84765625},{"t_s":47,"rss_mib":653.10546875},{"t_s":48,"rss_mib":653.10546875},{"t_s":49,"rss_mib":653.13671875},{"t_s":51,"rss_mib":653.13671875},{"t_s":53,"rss_mib":654.01171875},{"t_s":54,"rss_mib":654.01171875},{"t_s":56,"rss_mib":654.37890625},{"t_s":57,"rss_mib":654.37890625},{"t_s":60,"rss_mib":655.046875},{"t_s":60,"rss_mib":655.1953125},{"t_s":62,"rss_mib":656.26171875},{"t_s":63,"rss_mib":656.26171875},{"t_s":64,"rss_mib":656.265625},{"t_s":66,"rss_mib":656.265625},{"t_s":69,"rss_mib":656.265625},{"t_s":72,"rss_mib":656.265625},{"t_s":75,"rss_mib":656.265625},{"t_s":78,"rss_mib":656.265625},{"t_s":81,"rss_mib":656.2734375},{"t_s":84,"rss_mib":656.2734375},{"t_s":87,"rss_mib":656.2734375},{"t_s":90,"rss_mib":656.2734375},{"t_s":93,"rss_mib":656.2734375},{"t_s":96,"rss_mib":656.2734375},{"t_s":99,"rss_mib":656.2734375},{"t_s":102,"rss_mib":656.2734375},{"t_s":103,"rss_mib":656.27734375},{"t_s":105,"rss_mib":656.27734375},{"t_s":108,"rss_mib":656.27734375},{"t_s":110,"rss_mib":656.3203125},{"t_s":111,"rss_mib":656.3203125},{"t_s":114,"rss_mib":656.3203125},{"t_s":116,"rss_mib":656.32421875},{"t_s":117,"rss_mib":656.32421875},{"t_s":117,"rss_mib":656.33203125},{"t_s":120,"rss_mib":656.33203125},{"t_s":123,"rss_mib":656.33203125},{"t_s":124,"rss_mib":656.3671875},{"t_s":126,"rss_mib":656.3671875},{"t_s":129,"rss_mib":657.421875},{"t_s":130,"rss_mib":657.421875},{"t_s":131,"rss_mib":657.6640625},{"t_s":132,"rss_mib":657.6640625},{"t_s":135,"rss_mib":658.046875},{"t_s":136,"rss_mib":658.046875},{"t_s":137,"rss_mib":660.34765625},{"t_s":139,"rss_mib":660.34765625},{"t_s":140,"rss_mib":661.1328125},{"t_s":142,"rss_mib":661.1328125},{"t_s":144,"rss_mib":661.59375},{"t_s":145,"rss_mib":661.59375},{"t_s":148,"rss_mib":661.72265625},{"t_s":148,"rss_mib":661.84375},{"t_s":150,"rss_mib":663.078125},{"t_s":151,"rss_mib":663.078125},{"t_s":153,"rss_mib":663.265625},{"t_s":154,"rss_mib":663.265625},{"t_s":156,"rss_mib":663.26953125},{"t_s":157,"rss_mib":663.26953125},{"t_s":159,"rss_mib":663.28125},{"t_s":160,"rss_mib":663.28125},{"t_s":163,"rss_mib":663.28125},{"t_s":166,"rss_mib":663.28125},{"t_s":167,"rss_mib":663.28515625},{"t_s":169,"rss_mib":663.28515625},{"t_s":172,"rss_mib":663.28515625},{"t_s":175,"rss_mib":663.28515625},{"t_s":178,"rss_mib":663.28515625},{"t_s":181,"rss_mib":663.28515625},{"t_s":182,"rss_mib":663.2890625},{"t_s":184,"rss_mib":663.2890625},{"t_s":185,"rss_mib":663.3125},{"t_s":187,"rss_mib":663.3125},{"t_s":187,"rss_mib":663.31640625},{"t_s":190,"rss_mib":663.31640625},{"t_s":191,"rss_mib":663.3203125},{"t_s":193,"rss_mib":663.3203125},{"t_s":196,"rss_mib":663.3203125},{"t_s":199,"rss_mib":663.3203125},{"t_s":202,"rss_mib":663.3203125},{"t_s":204,"rss_mib":663.33203125},{"t_s":205,"rss_mib":663.33203125},{"t_s":208,"rss_mib":663.33203125},{"t_s":211,"rss_mib":663.33203125},{"t_s":214,"rss_mib":663.33203125},{"t_s":217,"rss_mib":663.33203125},{"t_s":220,"rss_mib":663.33203125},{"t_s":223,"rss_mib":663.33203125},{"t_s":226,"rss_mib":663.33203125},{"t_s":229,"rss_mib":663.33203125},{"t_s":232,"rss_mib":663.33203125},{"t_s":235,"rss_mib":663.33203125},{"t_s":238,"rss_mib":663.33203125},{"t_s":241,"rss_mib":663.33203125},{"t_s":244,"rss_mib":663.33203125},{"t_s":247,"rss_mib":663.33203125},{"t_s":250,"rss_mib":663.33203125},{"t_s":253,"rss_mib":663.33203125},{"t_s":256,"rss_mib":663.33203125},{"t_s":260,"rss_mib":663.33203125},{"t_s":262,"rss_mib":663.33203125},{"t_s":266,"rss_mib":663.33203125},{"t_s":269,"rss_mib":663.33203125},{"t_s":272,"rss_mib":663.33203125},{"t_s":275,"rss_mib":663.33203125},{"t_s":278,"rss_mib":663.33203125},{"t_s":281,"rss_mib":663.33203125},{"t_s":284,"rss_mib":663.33203125},{"t_s":287,"rss_mib":663.33203125},{"t_s":290,"rss_mib":663.33203125},{"t_s":293,"rss_mib":663.33203125},{"t_s":296,"rss_mib":663.33203125},{"t_s":299,"rss_mib":663.33203125},{"t_s":302,"rss_mib":663.33203125},{"t_s":305,"rss_mib":663.33203125},{"t_s":308,"rss_mib":663.33203125},{"t_s":311,"rss_mib":663.33203125},{"t_s":314,"rss_mib":663.33203125},{"t_s":317,"rss_mib":663.33203125},{"t_s":320,"rss_mib":663.33203125},{"t_s":323,"rss_mib":663.33203125},{"t_s":326,"rss_mib":663.33203125},{"t_s":329,"rss_mib":663.33203125},{"t_s":332,"rss_mib":663.33203125},{"t_s":335,"rss_mib":663.33203125},{"t_s":338,"rss_mib":663.33203125},{"t_s":341,"rss_mib":663.33203125},{"t_s":344,"rss_mib":663.33203125},{"t_s":347,"rss_mib":663.33203125},{"t_s":350,"rss_mib":663.33203125},{"t_s":353,"rss_mib":663.33203125},{"t_s":356,"rss_mib":663.33203125},{"t_s":359,"rss_mib":663.33203125},{"t_s":362,"rss_mib":663.33203125}],"idle_rss_series":[{"t_s":0,"rss_mib":592.81640625},{"t_s":0,"rss_mib":592.81640625},{"t_s":0,"rss_mib":592.81640625},{"t_s":1,"rss_mib":592.81640625},{"t_s":1,"rss_mib":592.81640625},{"t_s":2,"rss_mib":592.81640625},{"t_s":2,"rss_mib":592.81640625},{"t_s":3,"rss_mib":592.81640625},{"t_s":3,"rss_mib":592.81640625},{"t_s":4,"rss_mib":592.81640625},{"t_s":5,"rss_mib":592.81640625},{"t_s":5,"rss_mib":592.81640625},{"t_s":5,"rss_mib":592.81640625},{"t_s":6,"rss_mib":592.81640625},{"t_s":6,"rss_mib":592.81640625},{"t_s":7,"rss_mib":592.81640625},{"t_s":7,"rss_mib":592.81640625},{"t_s":8,"rss_mib":592.81640625},{"t_s":8,"rss_mib":592.81640625},{"t_s":9,"rss_mib":592.81640625},{"t_s":10,"rss_mib":592.81640625},{"t_s":10,"rss_mib":592.81640625},{"t_s":10,"rss_mib":592.81640625},{"t_s":11,"rss_mib":592.81640625},{"t_s":11,"rss_mib":592.81640625},{"t_s":12,"rss_mib":592.81640625},{"t_s":12,"rss_mib":592.81640625},{"t_s":13,"rss_mib":592.81640625},{"t_s":13,"rss_mib":592.81640625},{"t_s":14,"rss_mib":592.81640625},{"t_s":15,"rss_mib":592.81640625},{"t_s":15,"rss_mib":592.81640625},{"t_s":15,"rss_mib":592.81640625},{"t_s":16,"rss_mib":592.81640625},{"t_s":16,"rss_mib":592.81640625},{"t_s":17,"rss_mib":592.81640625},{"t_s":17,"rss_mib":592.81640625},{"t_s":18,"rss_mib":592.81640625},{"t_s":18,"rss_mib":592.81640625},{"t_s":19,"rss_mib":592.81640625},{"t_s":20,"rss_mib":592.81640625},{"t_s":20,"rss_mib":592.81640625},{"t_s":20,"rss_mib":592.81640625},{"t_s":21,"rss_mib":592.81640625},{"t_s":21,"rss_mib":592.81640625},{"t_s":22,"rss_mib":592.81640625},{"t_s":22,"rss_mib":592.81640625},{"t_s":23,"rss_mib":592.81640625},{"t_s":23,"rss_mib":592.81640625},{"t_s":24,"rss_mib":592.81640625},{"t_s":25,"rss_mib":592.81640625},{"t_s":25,"rss_mib":592.81640625},{"t_s":25,"rss_mib":592.81640625},{"t_s":26,"rss_mib":592.81640625},{"t_s":26,"rss_mib":592.81640625},{"t_s":27,"rss_mib":592.81640625},{"t_s":27,"rss_mib":592.81640625},{"t_s":28,"rss_mib":592.81640625},{"t_s":28,"rss_mib":592.81640625},{"t_s":29,"rss_mib":592.81640625},{"t_s":30,"rss_mib":592.81640625},{"t_s":30,"rss_mib":592.81640625},{"t_s":30,"rss_mib":592.81640625},{"t_s":31,"rss_mib":592.81640625},{"t_s":31,"rss_mib":592.81640625},{"t_s":32,"rss_mib":592.81640625},{"t_s":32,"rss_mib":592.81640625},{"t_s":33,"rss_mib":592.81640625},{"t_s":33,"rss_mib":592.81640625},{"t_s":34,"rss_mib":592.81640625},{"t_s":35,"rss_mib":592.81640625},{"t_s":35,"rss_mib":592.81640625},{"t_s":35,"rss_mib":592.81640625},{"t_s":36,"rss_mib":592.81640625},{"t_s":36,"rss_mib":592.81640625},{"t_s":37,"rss_mib":592.81640625},{"t_s":37,"rss_mib":592.81640625},{"t_s":38,"rss_mib":592.81640625},{"t_s":38,"rss_mib":592.81640625},{"t_s":39,"rss_mib":592.81640625},{"t_s":40,"rss_mib":592.81640625},{"t_s":40,"rss_mib":592.81640625},{"t_s":40,"rss_mib":592.81640625},{"t_s":41,"rss_mib":592.81640625},{"t_s":41,"rss_mib":592.81640625},{"t_s":42,"rss_mib":592.81640625},{"t_s":42,"rss_mib":592.81640625},{"t_s":43,"rss_mib":592.81640625},{"t_s":43,"rss_mib":592.81640625},{"t_s":44,"rss_mib":592.81640625},{"t_s":45,"rss_mib":592.81640625},{"t_s":45,"rss_mib":592.81640625},{"t_s":45,"rss_mib":592.81640625},{"t_s":46,"rss_mib":592.81640625},{"t_s":46,"rss_mib":592.81640625},{"t_s":47,"rss_mib":592.81640625},{"t_s":47,"rss_mib":592.81640625},{"t_s":48,"rss_mib":592.81640625},{"t_s":49,"rss_mib":592.81640625},{"t_s":49,"rss_mib":592.81640625},{"t_s":50,"rss_mib":592.81640625},{"t_s":50,"rss_mib":592.81640625},{"t_s":50,"rss_mib":592.81640625},{"t_s":51,"rss_mib":592.81640625},{"t_s":51,"rss_mib":592.81640625},{"t_s":52,"rss_mib":592.81640625},{"t_s":52,"rss_mib":592.81640625},{"t_s":53,"rss_mib":592.81640625},{"t_s":54,"rss_mib":592.81640625},{"t_s":54,"rss_mib":592.81640625},{"t_s":55,"rss_mib":592.81640625},{"t_s":55,"rss_mib":592.81640625},{"t_s":55,"rss_mib":592.81640625},{"t_s":56,"rss_mib":592.81640625},{"t_s":56,"rss_mib":592.81640625},{"t_s":57,"rss_mib":592.81640625},{"t_s":57,"rss_mib":592.81640625},{"t_s":58,"rss_mib":592.81640625},{"t_s":59,"rss_mib":592.81640625},{"t_s":59,"rss_mib":592.81640625},{"t_s":59,"rss_mib":592.81640625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.017867747,"cost":6.102982397,"memory":438.246156509,"streaming":0.000002358,"streams_sustained":0.000001906,"throughput":572.794505368},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}}},"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":7551,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":8314,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":8341,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":7593.984962406015,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":131.68316831683168,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2502569364427439,"certified":true,"suppressed":false},"cost_window_ok":{"value":798,"certified":true,"suppressed":false},"cost_window_rps":{"value":132,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.03132832080200501,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":2,"lower_bound":false,"samples":787},{"bound_ms":50,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":8,"lower_bound":false,"samples":787},{"bound_ms":100,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":16,"lower_bound":false,"samples":787},{"bound_ms":null,"rps":{"value":138,"certified":true,"suppressed":false},"concurrency":64,"p99_us":510607,"first_disqualified_conc":16384,"lower_bound":false,"samples":829}],"sweep":[{"conc":1,"ok":776,"rps":129,"p99_us":8508,"fail":0},{"conc":1,"ok":783,"rps":130,"p99_us":8417,"fail":0},{"conc":1,"ok":787,"rps":131,"p99_us":8409,"fail":0},{"conc":2,"ok":784,"rps":130,"p99_us":16280,"fail":0},{"conc":2,"ok":788,"rps":131,"p99_us":16118,"fail":0},{"conc":2,"ok":787,"rps":131,"p99_us":16403,"fail":0},{"conc":4,"ok":783,"rps":130,"p99_us":32617,"fail":0},{"conc":4,"ok":785,"rps":130,"p99_us":31863,"fail":0},{"conc":4,"ok":779,"rps":129,"p99_us":32369,"fail":0},{"conc":8,"ok":781,"rps":130,"p99_us":64987,"fail":0},{"conc":8,"ok":783,"rps":130,"p99_us":66235,"fail":0},{"conc":8,"ok":783,"rps":130,"p99_us":65825,"fail":0},{"conc":16,"ok":788,"rps":131,"p99_us":132178,"fail":0},{"conc":16,"ok":789,"rps":131,"p99_us":133406,"fail":0},{"conc":16,"ok":789,"rps":131,"p99_us":133940,"fail":0},{"conc":32,"ok":797,"rps":132,"p99_us":356272,"fail":0},{"conc":32,"ok":794,"rps":132,"p99_us":338697,"fail":0},{"conc":32,"ok":793,"rps":132,"p99_us":366565,"fail":0},{"conc":64,"ok":746,"rps":123,"p99_us":939079,"fail":0},{"conc":64,"ok":814,"rps":135,"p99_us":515904,"fail":0},{"conc":64,"ok":829,"rps":138,"p99_us":510607,"fail":0},{"conc":128,"ok":707,"rps":117,"p99_us":5259210,"fail":0},{"conc":128,"ok":765,"rps":127,"p99_us":4788836,"fail":0},{"conc":128,"ok":711,"rps":117,"p99_us":5233145,"fail":0},{"conc":256,"ok":759,"rps":126,"p99_us":5341012,"fail":0},{"conc":256,"ok":704,"rps":117,"p99_us":5445613,"fail":0},{"conc":256,"ok":641,"rps":106,"p99_us":5187951,"fail":0},{"conc":512,"ok":756,"rps":125,"p99_us":5315641,"fail":0},{"conc":512,"ok":647,"rps":107,"p99_us":4358299,"fail":0},{"conc":512,"ok":775,"rps":129,"p99_us":5397296,"fail":0},{"conc":1024,"ok":732,"rps":121,"p99_us":5112150,"fail":0},{"conc":1024,"ok":772,"rps":128,"p99_us":5311705,"fail":0},{"conc":1024,"ok":735,"rps":122,"p99_us":5176616,"fail":0},{"conc":2048,"ok":518,"rps":86,"p99_us":4408610,"fail":0},{"conc":2048,"ok":657,"rps":109,"p99_us":4953479,"fail":0},{"conc":2048,"ok":650,"rps":108,"p99_us":3595936,"fail":0},{"conc":4096,"ok":745,"rps":124,"p99_us":5259447,"fail":0},{"conc":4096,"ok":647,"rps":107,"p99_us":5474094,"fail":0},{"conc":4096,"ok":3,"rps":0.4999157392021575,"p99_us":5920550,"fail":504},{"conc":8192,"ok":209,"rps":34,"p99_us":5833359,"fail":0},{"conc":8192,"ok":126,"rps":20,"p99_us":5894128,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":3726},{"conc":16384,"ok":669,"rps":111,"p99_us":4588890,"fail":5645},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":1066}],"rungs":[{"conc":1,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":8417,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":131,"certified":true,"suppressed":false},"p99_us":16280,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":32369,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":65825,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":131,"certified":true,"suppressed":false},"p99_us":133406,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":132,"certified":true,"suppressed":false},"p99_us":356272,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":135,"certified":true,"suppressed":false},"p99_us":515904,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":5233145,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":5341012,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":125,"certified":true,"suppressed":false},"p99_us":5315641,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":5176616,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":108,"certified":true,"suppressed":false},"p99_us":4408610,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":107,"certified":true,"suppressed":false},"p99_us":5259447,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":20,"certified":true,"suppressed":false},"p99_us":5833359,"windows":3,"clean_windows":2},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 790 successful gateway round trip(s) and 267951 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":790,"direct_c1_samples":267951},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.56640625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":666.06640625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":666.0703125,"certified":true,"suppressed":false},"peak_rss_mib":{"value":666.0703125,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":666.0703125,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.0020496859180001844,"certified":true,"suppressed":false},"load_s":{"value":301,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.56640625},{"t_s":2,"rss_mib":665.43359375},{"t_s":2,"rss_mib":665.43359375},{"t_s":5,"rss_mib":665.59375},{"t_s":6,"rss_mib":665.6484375},{"t_s":8,"rss_mib":665.69140625},{"t_s":8,"rss_mib":665.69140625},{"t_s":11,"rss_mib":665.796875},{"t_s":12,"rss_mib":665.80859375},{"t_s":14,"rss_mib":665.89453125},{"t_s":15,"rss_mib":665.89453125},{"t_s":17,"rss_mib":665.96484375},{"t_s":18,"rss_mib":665.96484375},{"t_s":19,"rss_mib":665.98046875},{"t_s":21,"rss_mib":665.98046875},{"t_s":21,"rss_mib":665.984375},{"t_s":24,"rss_mib":665.984375},{"t_s":25,"rss_mib":665.98828125},{"t_s":27,"rss_mib":665.98828125},{"t_s":29,"rss_mib":665.99609375},{"t_s":30,"rss_mib":665.99609375},{"t_s":31,"rss_mib":666.00390625},{"t_s":33,"rss_mib":666.00390625},{"t_s":34,"rss_mib":666.0078125},{"t_s":36,"rss_mib":666.0078125},{"t_s":37,"rss_mib":666.01171875},{"t_s":39,"rss_mib":666.01171875},{"t_s":41,"rss_mib":666.015625},{"t_s":42,"rss_mib":666.01953125},{"t_s":45,"rss_mib":666.01953125},{"t_s":48,"rss_mib":666.01953125},{"t_s":51,"rss_mib":666.01953125},{"t_s":53,"rss_mib":666.03515625},{"t_s":54,"rss_mib":666.03515625},{"t_s":55,"rss_mib":666.0390625},{"t_s":57,"rss_mib":666.0390625},{"t_s":60,"rss_mib":666.0390625},{"t_s":63,"rss_mib":666.0390625},{"t_s":66,"rss_mib":666.0390625},{"t_s":69,"rss_mib":666.0390625},{"t_s":69,"rss_mib":666.04296875},{"t_s":72,"rss_mib":666.04296875},{"t_s":75,"rss_mib":666.04296875},{"t_s":78,"rss_mib":666.04296875},{"t_s":81,"rss_mib":666.04296875},{"t_s":84,"rss_mib":666.04296875},{"t_s":86,"rss_mib":666.046875},{"t_s":87,"rss_mib":666.046875},{"t_s":89,"rss_mib":666.05078125},{"t_s":90,"rss_mib":666.05078125},{"t_s":93,"rss_mib":666.05078125},{"t_s":96,"rss_mib":666.05078125},{"t_s":99,"rss_mib":666.05078125},{"t_s":102,"rss_mib":666.05078125},{"t_s":105,"rss_mib":666.05078125},{"t_s":108,"rss_mib":666.05078125},{"t_s":111,"rss_mib":666.05078125},{"t_s":114,"rss_mib":666.05078125},{"t_s":116,"rss_mib":666.0546875},{"t_s":117,"rss_mib":666.0546875},{"t_s":120,"rss_mib":666.0546875},{"t_s":123,"rss_mib":666.0546875},{"t_s":126,"rss_mib":666.0546875},{"t_s":129,"rss_mib":666.0546875},{"t_s":132,"rss_mib":666.0546875},{"t_s":135,"rss_mib":666.0546875},{"t_s":138,"rss_mib":666.0546875},{"t_s":141,"rss_mib":666.0546875},{"t_s":144,"rss_mib":666.0546875},{"t_s":147,"rss_mib":666.0546875},{"t_s":150,"rss_mib":666.0546875},{"t_s":153,"rss_mib":666.0546875},{"t_s":156,"rss_mib":666.0546875},{"t_s":159,"rss_mib":666.0546875},{"t_s":162,"rss_mib":666.0546875},{"t_s":165,"rss_mib":666.0546875},{"t_s":168,"rss_mib":666.0546875},{"t_s":171,"rss_mib":666.0546875},{"t_s":174,"rss_mib":666.0546875},{"t_s":177,"rss_mib":666.0546875},{"t_s":180,"rss_mib":666.0546875},{"t_s":183,"rss_mib":666.0546875},{"t_s":186,"rss_mib":666.0546875},{"t_s":189,"rss_mib":666.0546875},{"t_s":192,"rss_mib":666.0546875},{"t_s":195,"rss_mib":666.0546875},{"t_s":198,"rss_mib":666.0546875},{"t_s":201,"rss_mib":666.0546875},{"t_s":204,"rss_mib":666.0546875},{"t_s":207,"rss_mib":666.0546875},{"t_s":211,"rss_mib":666.0546875},{"t_s":213,"rss_mib":666.0546875},{"t_s":215,"rss_mib":666.06640625},{"t_s":217,"rss_mib":666.06640625},{"t_s":220,"rss_mib":666.06640625},{"t_s":223,"rss_mib":666.06640625},{"t_s":226,"rss_mib":666.06640625},{"t_s":229,"rss_mib":666.06640625},{"t_s":232,"rss_mib":666.06640625},{"t_s":235,"rss_mib":666.06640625},{"t_s":238,"rss_mib":666.06640625},{"t_s":241,"rss_mib":666.06640625},{"t_s":244,"rss_mib":666.06640625},{"t_s":247,"rss_mib":666.06640625},{"t_s":247,"rss_mib":666.0703125},{"t_s":250,"rss_mib":666.0703125},{"t_s":253,"rss_mib":666.0703125},{"t_s":256,"rss_mib":666.0703125},{"t_s":259,"rss_mib":666.0703125},{"t_s":262,"rss_mib":666.0703125},{"t_s":265,"rss_mib":666.0703125},{"t_s":268,"rss_mib":666.0703125},{"t_s":271,"rss_mib":666.0703125},{"t_s":274,"rss_mib":666.0703125},{"t_s":277,"rss_mib":666.0703125},{"t_s":280,"rss_mib":666.0703125},{"t_s":283,"rss_mib":666.0703125},{"t_s":286,"rss_mib":666.0703125},{"t_s":289,"rss_mib":666.0703125},{"t_s":292,"rss_mib":666.0703125},{"t_s":295,"rss_mib":666.0703125},{"t_s":298,"rss_mib":666.0703125},{"t_s":301,"rss_mib":666.0703125},{"t_s":304,"rss_mib":666.0703125},{"t_s":307,"rss_mib":666.0703125},{"t_s":310,"rss_mib":666.0703125},{"t_s":313,"rss_mib":666.0703125},{"t_s":316,"rss_mib":666.0703125},{"t_s":319,"rss_mib":666.0703125},{"t_s":322,"rss_mib":666.0703125},{"t_s":325,"rss_mib":666.0703125},{"t_s":328,"rss_mib":666.0703125},{"t_s":331,"rss_mib":666.0703125},{"t_s":334,"rss_mib":666.0703125},{"t_s":337,"rss_mib":666.0703125},{"t_s":340,"rss_mib":666.0703125},{"t_s":343,"rss_mib":666.0703125},{"t_s":346,"rss_mib":666.0703125},{"t_s":349,"rss_mib":666.0703125},{"t_s":352,"rss_mib":666.0703125},{"t_s":355,"rss_mib":666.0703125},{"t_s":358,"rss_mib":666.0703125},{"t_s":361,"rss_mib":666.0703125}],"idle_rss_series":[{"t_s":0,"rss_mib":592.56640625},{"t_s":0,"rss_mib":592.56640625},{"t_s":0,"rss_mib":592.56640625},{"t_s":1,"rss_mib":592.56640625},{"t_s":1,"rss_mib":592.56640625},{"t_s":2,"rss_mib":592.56640625},{"t_s":2,"rss_mib":592.56640625},{"t_s":3,"rss_mib":592.56640625},{"t_s":3,"rss_mib":592.56640625},{"t_s":4,"rss_mib":592.56640625},{"t_s":5,"rss_mib":592.56640625},{"t_s":5,"rss_mib":592.56640625},{"t_s":5,"rss_mib":592.56640625},{"t_s":6,"rss_mib":592.56640625},{"t_s":6,"rss_mib":592.56640625},{"t_s":7,"rss_mib":592.56640625},{"t_s":7,"rss_mib":592.56640625},{"t_s":8,"rss_mib":592.56640625},{"t_s":8,"rss_mib":592.56640625},{"t_s":9,"rss_mib":592.56640625},{"t_s":10,"rss_mib":592.56640625},{"t_s":10,"rss_mib":592.56640625},{"t_s":10,"rss_mib":592.56640625},{"t_s":11,"rss_mib":592.56640625},{"t_s":11,"rss_mib":592.56640625},{"t_s":12,"rss_mib":592.56640625},{"t_s":12,"rss_mib":592.56640625},{"t_s":13,"rss_mib":592.56640625},{"t_s":13,"rss_mib":592.56640625},{"t_s":14,"rss_mib":592.56640625},{"t_s":15,"rss_mib":592.56640625},{"t_s":15,"rss_mib":592.56640625},{"t_s":15,"rss_mib":592.56640625},{"t_s":16,"rss_mib":592.56640625},{"t_s":16,"rss_mib":592.56640625},{"t_s":17,"rss_mib":592.56640625},{"t_s":17,"rss_mib":592.56640625},{"t_s":18,"rss_mib":592.56640625},{"t_s":18,"rss_mib":592.56640625},{"t_s":19,"rss_mib":592.56640625},{"t_s":20,"rss_mib":592.56640625},{"t_s":20,"rss_mib":592.56640625},{"t_s":20,"rss_mib":592.56640625},{"t_s":21,"rss_mib":592.56640625},{"t_s":21,"rss_mib":592.56640625},{"t_s":22,"rss_mib":592.56640625},{"t_s":22,"rss_mib":592.56640625},{"t_s":23,"rss_mib":592.56640625},{"t_s":23,"rss_mib":592.56640625},{"t_s":24,"rss_mib":592.56640625},{"t_s":25,"rss_mib":592.56640625},{"t_s":25,"rss_mib":592.56640625},{"t_s":25,"rss_mib":592.56640625},{"t_s":26,"rss_mib":592.56640625},{"t_s":26,"rss_mib":592.56640625},{"t_s":27,"rss_mib":592.56640625},{"t_s":27,"rss_mib":592.56640625},{"t_s":28,"rss_mib":592.56640625},{"t_s":28,"rss_mib":592.56640625},{"t_s":29,"rss_mib":592.56640625},{"t_s":30,"rss_mib":592.56640625},{"t_s":30,"rss_mib":592.56640625},{"t_s":30,"rss_mib":592.56640625},{"t_s":31,"rss_mib":592.56640625},{"t_s":31,"rss_mib":592.56640625},{"t_s":32,"rss_mib":592.56640625},{"t_s":32,"rss_mib":592.56640625},{"t_s":33,"rss_mib":592.56640625},{"t_s":33,"rss_mib":592.56640625},{"t_s":34,"rss_mib":592.56640625},{"t_s":35,"rss_mib":592.56640625},{"t_s":35,"rss_mib":592.56640625},{"t_s":35,"rss_mib":592.56640625},{"t_s":36,"rss_mib":592.56640625},{"t_s":36,"rss_mib":592.56640625},{"t_s":37,"rss_mib":592.56640625},{"t_s":37,"rss_mib":592.56640625},{"t_s":38,"rss_mib":592.56640625},{"t_s":38,"rss_mib":592.56640625},{"t_s":39,"rss_mib":592.56640625},{"t_s":40,"rss_mib":592.56640625},{"t_s":40,"rss_mib":592.56640625},{"t_s":40,"rss_mib":592.56640625},{"t_s":41,"rss_mib":592.56640625},{"t_s":41,"rss_mib":592.56640625},{"t_s":42,"rss_mib":592.56640625},{"t_s":42,"rss_mib":592.56640625},{"t_s":43,"rss_mib":592.56640625},{"t_s":43,"rss_mib":592.56640625},{"t_s":44,"rss_mib":592.56640625},{"t_s":45,"rss_mib":592.56640625},{"t_s":45,"rss_mib":592.56640625},{"t_s":45,"rss_mib":592.56640625},{"t_s":46,"rss_mib":592.56640625},{"t_s":46,"rss_mib":592.56640625},{"t_s":47,"rss_mib":592.56640625},{"t_s":47,"rss_mib":592.56640625},{"t_s":48,"rss_mib":592.56640625},{"t_s":48,"rss_mib":592.56640625},{"t_s":49,"rss_mib":592.56640625},{"t_s":50,"rss_mib":592.56640625},{"t_s":50,"rss_mib":592.56640625},{"t_s":50,"rss_mib":592.56640625},{"t_s":51,"rss_mib":592.56640625},{"t_s":51,"rss_mib":592.56640625},{"t_s":52,"rss_mib":592.56640625},{"t_s":52,"rss_mib":592.56640625},{"t_s":53,"rss_mib":592.56640625},{"t_s":53,"rss_mib":592.56640625},{"t_s":54,"rss_mib":592.56640625},{"t_s":55,"rss_mib":592.56640625},{"t_s":55,"rss_mib":592.56640625},{"t_s":55,"rss_mib":592.56640625},{"t_s":56,"rss_mib":592.56640625},{"t_s":56,"rss_mib":592.56640625},{"t_s":57,"rss_mib":592.56640625},{"t_s":57,"rss_mib":592.56640625},{"t_s":58,"rss_mib":592.56640625},{"t_s":58,"rss_mib":592.56640625},{"t_s":59,"rss_mib":592.56640625},{"t_s":59,"rss_mib":592.56640625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.014971523,"cost":6.067439294,"memory":437.109574377,"streaming":0.000006955,"streams_sustained":0.000001976,"throughput":604.239429162},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}}},"bedrock":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/model/command-r/converse","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"anthropic":{"served":true,"reason":null,"status":"","path":"/v1/messages","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":8248,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":9054,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":9082,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":8407.202216066482,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":118.94563426688632,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2504942972604469,"certified":true,"suppressed":false},"cost_window_ok":{"value":722,"certified":true,"suppressed":false},"cost_window_rps":{"value":120,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.024930747922437674,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":118,"certified":true,"suppressed":false},"concurrency":1,"p99_us":9145,"first_disqualified_conc":2,"lower_bound":false,"samples":713},{"bound_ms":50,"rps":{"value":120,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17574,"first_disqualified_conc":8,"lower_bound":false,"samples":722},{"bound_ms":100,"rps":{"value":120,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17574,"first_disqualified_conc":16,"lower_bound":false,"samples":722},{"bound_ms":null,"rps":{"value":120,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17574,"first_disqualified_conc":16384,"lower_bound":false,"samples":722}],"sweep":[{"conc":1,"ok":713,"rps":118,"p99_us":9121,"fail":0},{"conc":1,"ok":713,"rps":118,"p99_us":9148,"fail":0},{"conc":1,"ok":713,"rps":118,"p99_us":9145,"fail":0},{"conc":2,"ok":716,"rps":119,"p99_us":17675,"fail":0},{"conc":2,"ok":720,"rps":119,"p99_us":17771,"fail":0},{"conc":2,"ok":722,"rps":120,"p99_us":17574,"fail":0},{"conc":4,"ok":716,"rps":119,"p99_us":36055,"fail":0},{"conc":4,"ok":718,"rps":119,"p99_us":35456,"fail":0},{"conc":4,"ok":715,"rps":119,"p99_us":35966,"fail":0},{"conc":8,"ok":715,"rps":119,"p99_us":74030,"fail":0},{"conc":8,"ok":716,"rps":119,"p99_us":71151,"fail":0},{"conc":8,"ok":717,"rps":119,"p99_us":71564,"fail":0},{"conc":16,"ok":722,"rps":120,"p99_us":143427,"fail":0},{"conc":16,"ok":715,"rps":119,"p99_us":144870,"fail":0},{"conc":16,"ok":663,"rps":110,"p99_us":570623,"fail":0},{"conc":32,"ok":714,"rps":118,"p99_us":371262,"fail":0},{"conc":32,"ok":712,"rps":118,"p99_us":366906,"fail":0},{"conc":32,"ok":719,"rps":119,"p99_us":389463,"fail":0},{"conc":64,"ok":724,"rps":120,"p99_us":566600,"fail":0},{"conc":64,"ok":712,"rps":118,"p99_us":1267583,"fail":0},{"conc":64,"ok":694,"rps":115,"p99_us":990372,"fail":0},{"conc":128,"ok":640,"rps":105,"p99_us":1127837,"fail":0},{"conc":128,"ok":706,"rps":117,"p99_us":4613511,"fail":0},{"conc":128,"ok":640,"rps":105,"p99_us":5130336,"fail":0},{"conc":256,"ok":567,"rps":94,"p99_us":2547402,"fail":0},{"conc":256,"ok":676,"rps":112,"p99_us":5581175,"fail":0},{"conc":256,"ok":615,"rps":102,"p99_us":2228756,"fail":0},{"conc":512,"ok":631,"rps":105,"p99_us":5586216,"fail":0},{"conc":512,"ok":641,"rps":106,"p99_us":4919521,"fail":0},{"conc":512,"ok":654,"rps":108,"p99_us":5066911,"fail":0},{"conc":1024,"ok":682,"rps":113,"p99_us":4685287,"fail":0},{"conc":1024,"ok":631,"rps":105,"p99_us":4550973,"fail":0},{"conc":1024,"ok":604,"rps":100,"p99_us":4285840,"fail":0},{"conc":2048,"ok":620,"rps":103,"p99_us":4579093,"fail":0},{"conc":2048,"ok":634,"rps":105,"p99_us":4824531,"fail":0},{"conc":2048,"ok":636,"rps":105,"p99_us":4795904,"fail":0},{"conc":4096,"ok":614,"rps":102,"p99_us":4310121,"fail":0},{"conc":4096,"ok":502,"rps":83,"p99_us":5547826,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":3804},{"conc":8192,"ok":675,"rps":112,"p99_us":5397769,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":12287},{"conc":16384,"ok":481,"rps":80,"p99_us":5187090,"fail":6245},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":2386}],"rungs":[{"conc":1,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":9145,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":17675,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":35966,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":71564,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":144870,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":371262,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":990372,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":105,"certified":true,"suppressed":false},"p99_us":4613511,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":2547402,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":106,"certified":true,"suppressed":false},"p99_us":5066911,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":105,"certified":true,"suppressed":false},"p99_us":4550973,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":105,"certified":true,"suppressed":false},"p99_us":4795904,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":4310121,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":112,"certified":true,"suppressed":false},"p99_us":5397769,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 723 successful gateway round trip(s) and 260556 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":723,"direct_c1_samples":260556},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.1796875,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":665.765625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":665.765625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":665.765625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":665.765625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.1796875},{"t_s":1,"rss_mib":665.765625},{"t_s":2,"rss_mib":665.765625},{"t_s":6,"rss_mib":665.765625},{"t_s":8,"rss_mib":665.765625},{"t_s":12,"rss_mib":665.765625},{"t_s":15,"rss_mib":665.765625},{"t_s":18,"rss_mib":665.765625},{"t_s":21,"rss_mib":665.765625},{"t_s":24,"rss_mib":665.765625},{"t_s":27,"rss_mib":665.765625},{"t_s":30,"rss_mib":665.765625},{"t_s":33,"rss_mib":665.765625},{"t_s":36,"rss_mib":665.765625},{"t_s":39,"rss_mib":665.765625},{"t_s":42,"rss_mib":665.765625},{"t_s":45,"rss_mib":665.765625},{"t_s":48,"rss_mib":665.765625},{"t_s":51,"rss_mib":665.765625},{"t_s":54,"rss_mib":665.765625},{"t_s":57,"rss_mib":665.765625},{"t_s":60,"rss_mib":665.765625},{"t_s":63,"rss_mib":665.765625},{"t_s":66,"rss_mib":665.765625},{"t_s":69,"rss_mib":665.765625},{"t_s":72,"rss_mib":665.765625},{"t_s":75,"rss_mib":665.765625},{"t_s":78,"rss_mib":665.765625},{"t_s":81,"rss_mib":665.765625},{"t_s":84,"rss_mib":665.765625},{"t_s":87,"rss_mib":665.765625},{"t_s":90,"rss_mib":665.765625},{"t_s":93,"rss_mib":665.765625},{"t_s":96,"rss_mib":665.765625},{"t_s":99,"rss_mib":665.765625},{"t_s":102,"rss_mib":665.765625},{"t_s":105,"rss_mib":665.765625},{"t_s":108,"rss_mib":665.765625},{"t_s":111,"rss_mib":665.765625},{"t_s":114,"rss_mib":665.765625},{"t_s":117,"rss_mib":665.765625},{"t_s":120,"rss_mib":665.765625},{"t_s":123,"rss_mib":665.765625},{"t_s":126,"rss_mib":665.765625},{"t_s":129,"rss_mib":665.765625},{"t_s":132,"rss_mib":665.765625},{"t_s":135,"rss_mib":665.765625},{"t_s":138,"rss_mib":665.765625},{"t_s":141,"rss_mib":665.765625},{"t_s":144,"rss_mib":665.765625},{"t_s":147,"rss_mib":665.765625},{"t_s":150,"rss_mib":665.765625},{"t_s":153,"rss_mib":665.765625},{"t_s":156,"rss_mib":665.765625},{"t_s":159,"rss_mib":665.765625},{"t_s":162,"rss_mib":665.765625},{"t_s":165,"rss_mib":665.765625},{"t_s":169,"rss_mib":665.765625},{"t_s":171,"rss_mib":665.765625},{"t_s":175,"rss_mib":665.765625},{"t_s":178,"rss_mib":665.765625},{"t_s":181,"rss_mib":665.765625},{"t_s":184,"rss_mib":665.765625},{"t_s":187,"rss_mib":665.765625},{"t_s":190,"rss_mib":665.765625},{"t_s":193,"rss_mib":665.765625},{"t_s":196,"rss_mib":665.765625},{"t_s":199,"rss_mib":665.765625},{"t_s":202,"rss_mib":665.765625},{"t_s":205,"rss_mib":665.765625},{"t_s":208,"rss_mib":665.765625},{"t_s":211,"rss_mib":665.765625},{"t_s":214,"rss_mib":665.765625},{"t_s":217,"rss_mib":665.765625},{"t_s":220,"rss_mib":665.765625},{"t_s":223,"rss_mib":665.765625},{"t_s":226,"rss_mib":665.765625},{"t_s":229,"rss_mib":665.765625},{"t_s":232,"rss_mib":665.765625},{"t_s":235,"rss_mib":665.765625},{"t_s":238,"rss_mib":665.765625},{"t_s":241,"rss_mib":665.765625},{"t_s":244,"rss_mib":665.765625},{"t_s":247,"rss_mib":665.765625},{"t_s":250,"rss_mib":665.765625},{"t_s":253,"rss_mib":665.765625},{"t_s":256,"rss_mib":665.765625},{"t_s":259,"rss_mib":665.765625},{"t_s":262,"rss_mib":665.765625},{"t_s":265,"rss_mib":665.765625},{"t_s":268,"rss_mib":665.765625},{"t_s":271,"rss_mib":665.765625},{"t_s":274,"rss_mib":665.765625},{"t_s":277,"rss_mib":665.765625},{"t_s":280,"rss_mib":665.765625},{"t_s":283,"rss_mib":665.765625},{"t_s":286,"rss_mib":665.765625},{"t_s":289,"rss_mib":665.765625},{"t_s":292,"rss_mib":665.765625},{"t_s":295,"rss_mib":665.765625},{"t_s":298,"rss_mib":665.765625},{"t_s":301,"rss_mib":665.765625},{"t_s":304,"rss_mib":665.765625},{"t_s":307,"rss_mib":665.765625},{"t_s":310,"rss_mib":665.765625},{"t_s":313,"rss_mib":665.765625},{"t_s":316,"rss_mib":665.765625},{"t_s":319,"rss_mib":665.765625},{"t_s":322,"rss_mib":665.765625},{"t_s":325,"rss_mib":665.765625},{"t_s":328,"rss_mib":665.765625},{"t_s":331,"rss_mib":665.765625},{"t_s":335,"rss_mib":665.765625},{"t_s":337,"rss_mib":665.765625},{"t_s":341,"rss_mib":665.765625},{"t_s":344,"rss_mib":665.765625},{"t_s":347,"rss_mib":665.765625},{"t_s":350,"rss_mib":665.765625},{"t_s":353,"rss_mib":665.765625},{"t_s":356,"rss_mib":665.765625},{"t_s":359,"rss_mib":665.765625},{"t_s":362,"rss_mib":665.765625}],"idle_rss_series":[{"t_s":0,"rss_mib":592.1796875},{"t_s":0,"rss_mib":592.1796875},{"t_s":0,"rss_mib":592.1796875},{"t_s":1,"rss_mib":592.1796875},{"t_s":1,"rss_mib":592.1796875},{"t_s":2,"rss_mib":592.1796875},{"t_s":2,"rss_mib":592.1796875},{"t_s":3,"rss_mib":592.1796875},{"t_s":3,"rss_mib":592.1796875},{"t_s":4,"rss_mib":592.1796875},{"t_s":5,"rss_mib":592.1796875},{"t_s":5,"rss_mib":592.1796875},{"t_s":5,"rss_mib":592.1796875},{"t_s":6,"rss_mib":592.1796875},{"t_s":6,"rss_mib":592.1796875},{"t_s":7,"rss_mib":592.1796875},{"t_s":7,"rss_mib":592.1796875},{"t_s":8,"rss_mib":592.1796875},{"t_s":8,"rss_mib":592.1796875},{"t_s":9,"rss_mib":592.1796875},{"t_s":10,"rss_mib":592.1796875},{"t_s":10,"rss_mib":592.1796875},{"t_s":10,"rss_mib":592.1796875},{"t_s":11,"rss_mib":592.1796875},{"t_s":11,"rss_mib":592.1796875},{"t_s":12,"rss_mib":592.1796875},{"t_s":12,"rss_mib":592.1796875},{"t_s":13,"rss_mib":592.1796875},{"t_s":13,"rss_mib":592.1796875},{"t_s":14,"rss_mib":592.1796875},{"t_s":15,"rss_mib":592.1796875},{"t_s":15,"rss_mib":592.1796875},{"t_s":15,"rss_mib":592.1796875},{"t_s":16,"rss_mib":592.1796875},{"t_s":16,"rss_mib":592.1796875},{"t_s":17,"rss_mib":592.1796875},{"t_s":17,"rss_mib":592.1796875},{"t_s":18,"rss_mib":592.1796875},{"t_s":18,"rss_mib":592.1796875},{"t_s":19,"rss_mib":592.1796875},{"t_s":20,"rss_mib":592.1796875},{"t_s":20,"rss_mib":592.1796875},{"t_s":20,"rss_mib":592.1796875},{"t_s":21,"rss_mib":592.1796875},{"t_s":21,"rss_mib":592.1796875},{"t_s":22,"rss_mib":592.1796875},{"t_s":22,"rss_mib":592.1796875},{"t_s":23,"rss_mib":592.1796875},{"t_s":23,"rss_mib":592.1796875},{"t_s":24,"rss_mib":592.1796875},{"t_s":25,"rss_mib":592.1796875},{"t_s":25,"rss_mib":592.1796875},{"t_s":25,"rss_mib":592.1796875},{"t_s":26,"rss_mib":592.1796875},{"t_s":26,"rss_mib":592.1796875},{"t_s":27,"rss_mib":592.1796875},{"t_s":27,"rss_mib":592.1796875},{"t_s":28,"rss_mib":592.1796875},{"t_s":28,"rss_mib":592.1796875},{"t_s":29,"rss_mib":592.1796875},{"t_s":30,"rss_mib":592.1796875},{"t_s":30,"rss_mib":592.1796875},{"t_s":30,"rss_mib":592.1796875},{"t_s":31,"rss_mib":592.1796875},{"t_s":31,"rss_mib":592.1796875},{"t_s":32,"rss_mib":592.1796875},{"t_s":32,"rss_mib":592.1796875},{"t_s":33,"rss_mib":592.1796875},{"t_s":33,"rss_mib":592.1796875},{"t_s":34,"rss_mib":592.1796875},{"t_s":35,"rss_mib":592.1796875},{"t_s":35,"rss_mib":592.1796875},{"t_s":35,"rss_mib":592.1796875},{"t_s":36,"rss_mib":592.1796875},{"t_s":36,"rss_mib":592.1796875},{"t_s":37,"rss_mib":592.1796875},{"t_s":37,"rss_mib":592.1796875},{"t_s":38,"rss_mib":592.1796875},{"t_s":38,"rss_mib":592.1796875},{"t_s":39,"rss_mib":592.1796875},{"t_s":40,"rss_mib":592.1796875},{"t_s":40,"rss_mib":592.1796875},{"t_s":40,"rss_mib":592.1796875},{"t_s":41,"rss_mib":592.1796875},{"t_s":41,"rss_mib":592.1796875},{"t_s":42,"rss_mib":592.1796875},{"t_s":42,"rss_mib":592.1796875},{"t_s":43,"rss_mib":592.1796875},{"t_s":43,"rss_mib":592.1796875},{"t_s":44,"rss_mib":592.1796875},{"t_s":45,"rss_mib":592.1796875},{"t_s":45,"rss_mib":592.1796875},{"t_s":45,"rss_mib":592.1796875},{"t_s":46,"rss_mib":592.1796875},{"t_s":46,"rss_mib":592.1796875},{"t_s":47,"rss_mib":592.1796875},{"t_s":47,"rss_mib":592.1796875},{"t_s":48,"rss_mib":592.1796875},{"t_s":48,"rss_mib":592.1796875},{"t_s":49,"rss_mib":592.1796875},{"t_s":50,"rss_mib":592.1796875},{"t_s":50,"rss_mib":592.1796875},{"t_s":50,"rss_mib":592.1796875},{"t_s":51,"rss_mib":592.1796875},{"t_s":51,"rss_mib":592.1796875},{"t_s":52,"rss_mib":592.1796875},{"t_s":52,"rss_mib":592.1796875},{"t_s":53,"rss_mib":592.1796875},{"t_s":53,"rss_mib":592.1796875},{"t_s":54,"rss_mib":592.1796875},{"t_s":55,"rss_mib":592.1796875},{"t_s":55,"rss_mib":592.1796875},{"t_s":55,"rss_mib":592.1796875},{"t_s":56,"rss_mib":592.1796875},{"t_s":56,"rss_mib":592.1796875},{"t_s":57,"rss_mib":592.1796875},{"t_s":57,"rss_mib":592.1796875},{"t_s":58,"rss_mib":592.1796875},{"t_s":59,"rss_mib":592.1796875},{"t_s":59,"rss_mib":592.1796875},{"t_s":59,"rss_mib":592.1796875}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.011629189,"cost":6.07100536,"memory":437.615256215,"streaming":0.000002268,"streams_sustained":0.000002041,"throughput":624.677395041},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here"}}},"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":9710,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":10423,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":10451,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":9886.363636363636,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":101.14942528735632,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25085857203068684,"certified":true,"suppressed":false},"cost_window_ok":{"value":616,"certified":true,"suppressed":false},"cost_window_rps":{"value":102,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.04707792207792208,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":104,"certified":true,"suppressed":false},"concurrency":2,"p99_us":20420,"first_disqualified_conc":8,"lower_bound":false,"samples":630},{"bound_ms":100,"rps":{"value":104,"certified":true,"suppressed":false},"concurrency":2,"p99_us":20420,"first_disqualified_conc":16,"lower_bound":false,"samples":630},{"bound_ms":null,"rps":{"value":106,"certified":true,"suppressed":false},"concurrency":64,"p99_us":647637,"first_disqualified_conc":8192,"lower_bound":false,"samples":637}],"sweep":[{"conc":1,"ok":613,"rps":102,"p99_us":10403,"fail":0},{"conc":1,"ok":614,"rps":102,"p99_us":10436,"fail":0},{"conc":1,"ok":618,"rps":102,"p99_us":10376,"fail":0},{"conc":2,"ok":630,"rps":104,"p99_us":20420,"fail":0},{"conc":2,"ok":622,"rps":103,"p99_us":20191,"fail":0},{"conc":2,"ok":622,"rps":103,"p99_us":20116,"fail":0},{"conc":4,"ok":623,"rps":102,"p99_us":41399,"fail":0},{"conc":4,"ok":619,"rps":102,"p99_us":41570,"fail":0},{"conc":4,"ok":621,"rps":102,"p99_us":41240,"fail":0},{"conc":8,"ok":620,"rps":103,"p99_us":81984,"fail":0},{"conc":8,"ok":622,"rps":103,"p99_us":82618,"fail":0},{"conc":8,"ok":622,"rps":103,"p99_us":82679,"fail":0},{"conc":16,"ok":622,"rps":103,"p99_us":167399,"fail":0},{"conc":16,"ok":623,"rps":102,"p99_us":165252,"fail":0},{"conc":16,"ok":623,"rps":102,"p99_us":164697,"fail":0},{"conc":32,"ok":623,"rps":102,"p99_us":444832,"fail":0},{"conc":32,"ok":625,"rps":104,"p99_us":422182,"fail":0},{"conc":32,"ok":621,"rps":102,"p99_us":433554,"fail":0},{"conc":64,"ok":622,"rps":103,"p99_us":1494156,"fail":0},{"conc":64,"ok":637,"rps":106,"p99_us":647637,"fail":0},{"conc":64,"ok":583,"rps":97,"p99_us":1076394,"fail":0},{"conc":128,"ok":603,"rps":100,"p99_us":5426849,"fail":0},{"conc":128,"ok":551,"rps":91,"p99_us":1657226,"fail":0},{"conc":128,"ok":590,"rps":98,"p99_us":5545477,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":2825668,"fail":0},{"conc":256,"ok":536,"rps":89,"p99_us":5471548,"fail":0},{"conc":256,"ok":600,"rps":99,"p99_us":5432965,"fail":0},{"conc":512,"ok":528,"rps":87,"p99_us":4641645,"fail":0},{"conc":512,"ok":548,"rps":91,"p99_us":4796063,"fail":0},{"conc":512,"ok":559,"rps":93,"p99_us":4741681,"fail":0},{"conc":1024,"ok":373,"rps":61,"p99_us":5192411,"fail":0},{"conc":1024,"ok":540,"rps":89,"p99_us":5493077,"fail":0},{"conc":1024,"ok":593,"rps":98,"p99_us":5535117,"fail":0},{"conc":2048,"ok":383,"rps":63,"p99_us":5215509,"fail":0},{"conc":2048,"ok":588,"rps":97,"p99_us":5506387,"fail":0},{"conc":2048,"ok":380,"rps":63,"p99_us":5072971,"fail":0},{"conc":4096,"ok":369,"rps":61,"p99_us":5056404,"fail":656},{"conc":4096,"ok":38,"rps":6,"p99_us":5926850,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":181},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":10403,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":103,"certified":true,"suppressed":false},"p99_us":20191,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":41399,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":103,"certified":true,"suppressed":false},"p99_us":82618,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":165252,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":433554,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":103,"certified":true,"suppressed":false},"p99_us":1076394,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":98,"certified":true,"suppressed":false},"p99_us":5426849,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":5432965,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":91,"certified":true,"suppressed":false},"p99_us":4741681,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":5493077,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":5215509,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":6,"certified":true,"suppressed":false},"p99_us":5926850,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 1 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 615 successful gateway round trip(s) and 263786 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":615,"direct_c1_samples":263786},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.70703125,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":667.8046875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":304,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.70703125},{"t_s":2,"rss_mib":667.8046875},{"t_s":2,"rss_mib":667.8046875},{"t_s":6,"rss_mib":667.8046875},{"t_s":9,"rss_mib":667.8046875},{"t_s":12,"rss_mib":667.8046875},{"t_s":15,"rss_mib":667.8046875},{"t_s":18,"rss_mib":667.8046875},{"t_s":21,"rss_mib":667.8046875},{"t_s":24,"rss_mib":667.8046875},{"t_s":27,"rss_mib":667.8046875},{"t_s":30,"rss_mib":667.8046875},{"t_s":33,"rss_mib":667.8046875},{"t_s":36,"rss_mib":667.8046875},{"t_s":39,"rss_mib":667.8046875},{"t_s":42,"rss_mib":667.8046875},{"t_s":45,"rss_mib":667.8046875},{"t_s":48,"rss_mib":667.8046875},{"t_s":51,"rss_mib":667.8046875},{"t_s":54,"rss_mib":667.8046875},{"t_s":57,"rss_mib":667.8046875},{"t_s":60,"rss_mib":667.8046875},{"t_s":63,"rss_mib":667.8046875},{"t_s":66,"rss_mib":667.8046875},{"t_s":69,"rss_mib":667.8046875},{"t_s":72,"rss_mib":667.8046875},{"t_s":75,"rss_mib":667.8046875},{"t_s":78,"rss_mib":667.8046875},{"t_s":81,"rss_mib":667.8046875},{"t_s":84,"rss_mib":667.8046875},{"t_s":87,"rss_mib":667.8046875},{"t_s":91,"rss_mib":667.8046875},{"t_s":94,"rss_mib":667.8046875},{"t_s":97,"rss_mib":667.8046875},{"t_s":100,"rss_mib":667.8046875},{"t_s":103,"rss_mib":667.8046875},{"t_s":106,"rss_mib":667.8046875},{"t_s":109,"rss_mib":667.8046875},{"t_s":112,"rss_mib":667.8046875},{"t_s":115,"rss_mib":667.8046875},{"t_s":118,"rss_mib":667.8046875},{"t_s":121,"rss_mib":667.8046875},{"t_s":124,"rss_mib":667.8046875},{"t_s":127,"rss_mib":667.8046875},{"t_s":130,"rss_mib":667.8046875},{"t_s":133,"rss_mib":667.8046875},{"t_s":136,"rss_mib":667.8046875},{"t_s":139,"rss_mib":667.8046875},{"t_s":142,"rss_mib":667.8046875},{"t_s":145,"rss_mib":667.8046875},{"t_s":148,"rss_mib":667.8046875},{"t_s":151,"rss_mib":667.8046875},{"t_s":154,"rss_mib":667.8046875},{"t_s":157,"rss_mib":667.8046875},{"t_s":160,"rss_mib":667.8046875},{"t_s":163,"rss_mib":667.8046875},{"t_s":166,"rss_mib":667.8046875},{"t_s":169,"rss_mib":667.8046875},{"t_s":172,"rss_mib":667.8046875},{"t_s":175,"rss_mib":667.8046875},{"t_s":179,"rss_mib":667.8046875},{"t_s":182,"rss_mib":667.8046875},{"t_s":185,"rss_mib":667.8046875},{"t_s":188,"rss_mib":667.8046875},{"t_s":191,"rss_mib":667.8046875},{"t_s":194,"rss_mib":667.8046875},{"t_s":197,"rss_mib":667.8046875},{"t_s":200,"rss_mib":667.8046875},{"t_s":203,"rss_mib":667.8046875},{"t_s":206,"rss_mib":667.8046875},{"t_s":209,"rss_mib":667.8046875},{"t_s":212,"rss_mib":667.8046875},{"t_s":215,"rss_mib":667.8046875},{"t_s":218,"rss_mib":667.8046875},{"t_s":221,"rss_mib":667.8046875},{"t_s":224,"rss_mib":667.8046875},{"t_s":227,"rss_mib":667.8046875},{"t_s":230,"rss_mib":667.8046875},{"t_s":233,"rss_mib":667.8046875},{"t_s":236,"rss_mib":667.8046875},{"t_s":239,"rss_mib":667.8046875},{"t_s":242,"rss_mib":667.8046875},{"t_s":245,"rss_mib":667.8046875},{"t_s":248,"rss_mib":667.8046875},{"t_s":251,"rss_mib":667.8046875},{"t_s":254,"rss_mib":667.8046875},{"t_s":257,"rss_mib":667.8046875},{"t_s":260,"rss_mib":667.8046875},{"t_s":263,"rss_mib":667.8046875},{"t_s":266,"rss_mib":667.8046875},{"t_s":270,"rss_mib":667.8046875},{"t_s":273,"rss_mib":667.8046875},{"t_s":276,"rss_mib":667.8046875},{"t_s":279,"rss_mib":667.8046875},{"t_s":282,"rss_mib":667.8046875},{"t_s":285,"rss_mib":667.8046875},{"t_s":288,"rss_mib":667.8046875},{"t_s":291,"rss_mib":667.8046875},{"t_s":294,"rss_mib":667.8046875},{"t_s":297,"rss_mib":667.8046875},{"t_s":300,"rss_mib":667.8046875},{"t_s":303,"rss_mib":667.8046875},{"t_s":306,"rss_mib":667.8046875},{"t_s":309,"rss_mib":667.8046875},{"t_s":312,"rss_mib":667.8046875},{"t_s":315,"rss_mib":667.8046875},{"t_s":318,"rss_mib":667.8046875},{"t_s":321,"rss_mib":667.8046875},{"t_s":324,"rss_mib":667.8046875},{"t_s":327,"rss_mib":667.8046875},{"t_s":330,"rss_mib":667.8046875},{"t_s":333,"rss_mib":667.8046875},{"t_s":336,"rss_mib":667.8046875},{"t_s":339,"rss_mib":667.8046875},{"t_s":342,"rss_mib":667.8046875},{"t_s":345,"rss_mib":667.8046875},{"t_s":348,"rss_mib":667.8046875},{"t_s":351,"rss_mib":667.8046875},{"t_s":354,"rss_mib":667.8046875},{"t_s":357,"rss_mib":667.8046875},{"t_s":361,"rss_mib":667.8046875},{"t_s":363,"rss_mib":667.8046875}],"idle_rss_series":[{"t_s":0,"rss_mib":592.70703125},{"t_s":0,"rss_mib":592.70703125},{"t_s":0,"rss_mib":592.70703125},{"t_s":1,"rss_mib":592.70703125},{"t_s":1,"rss_mib":592.70703125},{"t_s":2,"rss_mib":592.70703125},{"t_s":2,"rss_mib":592.70703125},{"t_s":3,"rss_mib":592.70703125},{"t_s":3,"rss_mib":592.70703125},{"t_s":4,"rss_mib":592.70703125},{"t_s":5,"rss_mib":592.70703125},{"t_s":5,"rss_mib":592.70703125},{"t_s":5,"rss_mib":592.70703125},{"t_s":6,"rss_mib":592.70703125},{"t_s":6,"rss_mib":592.70703125},{"t_s":7,"rss_mib":592.70703125},{"t_s":7,"rss_mib":592.70703125},{"t_s":8,"rss_mib":592.70703125},{"t_s":8,"rss_mib":592.70703125},{"t_s":9,"rss_mib":592.70703125},{"t_s":10,"rss_mib":592.70703125},{"t_s":10,"rss_mib":592.70703125},{"t_s":10,"rss_mib":592.70703125},{"t_s":11,"rss_mib":592.70703125},{"t_s":11,"rss_mib":592.70703125},{"t_s":12,"rss_mib":592.70703125},{"t_s":12,"rss_mib":592.70703125},{"t_s":13,"rss_mib":592.70703125},{"t_s":13,"rss_mib":592.70703125},{"t_s":14,"rss_mib":592.70703125},{"t_s":15,"rss_mib":592.70703125},{"t_s":15,"rss_mib":592.70703125},{"t_s":15,"rss_mib":592.70703125},{"t_s":16,"rss_mib":592.70703125},{"t_s":16,"rss_mib":592.70703125},{"t_s":17,"rss_mib":592.70703125},{"t_s":17,"rss_mib":592.70703125},{"t_s":18,"rss_mib":592.70703125},{"t_s":18,"rss_mib":592.70703125},{"t_s":19,"rss_mib":592.70703125},{"t_s":20,"rss_mib":592.70703125},{"t_s":20,"rss_mib":592.70703125},{"t_s":20,"rss_mib":592.70703125},{"t_s":21,"rss_mib":592.70703125},{"t_s":21,"rss_mib":592.70703125},{"t_s":22,"rss_mib":592.70703125},{"t_s":22,"rss_mib":592.70703125},{"t_s":23,"rss_mib":592.70703125},{"t_s":23,"rss_mib":592.70703125},{"t_s":24,"rss_mib":592.70703125},{"t_s":25,"rss_mib":592.70703125},{"t_s":25,"rss_mib":592.70703125},{"t_s":25,"rss_mib":592.70703125},{"t_s":26,"rss_mib":592.70703125},{"t_s":26,"rss_mib":592.70703125},{"t_s":27,"rss_mib":592.70703125},{"t_s":27,"rss_mib":592.70703125},{"t_s":28,"rss_mib":592.70703125},{"t_s":28,"rss_mib":592.70703125},{"t_s":29,"rss_mib":592.70703125},{"t_s":30,"rss_mib":592.70703125},{"t_s":30,"rss_mib":592.70703125},{"t_s":30,"rss_mib":592.70703125},{"t_s":31,"rss_mib":592.70703125},{"t_s":31,"rss_mib":592.70703125},{"t_s":32,"rss_mib":592.70703125},{"t_s":32,"rss_mib":592.70703125},{"t_s":33,"rss_mib":592.70703125},{"t_s":33,"rss_mib":592.70703125},{"t_s":34,"rss_mib":592.70703125},{"t_s":35,"rss_mib":592.70703125},{"t_s":35,"rss_mib":592.70703125},{"t_s":35,"rss_mib":592.70703125},{"t_s":36,"rss_mib":592.70703125},{"t_s":36,"rss_mib":592.70703125},{"t_s":37,"rss_mib":592.70703125},{"t_s":37,"rss_mib":592.70703125},{"t_s":38,"rss_mib":592.70703125},{"t_s":38,"rss_mib":592.70703125},{"t_s":39,"rss_mib":592.70703125},{"t_s":40,"rss_mib":592.70703125},{"t_s":40,"rss_mib":592.70703125},{"t_s":40,"rss_mib":592.70703125},{"t_s":41,"rss_mib":592.70703125},{"t_s":41,"rss_mib":592.70703125},{"t_s":42,"rss_mib":592.70703125},{"t_s":42,"rss_mib":592.70703125},{"t_s":43,"rss_mib":592.70703125},{"t_s":44,"rss_mib":592.70703125},{"t_s":44,"rss_mib":592.70703125},{"t_s":45,"rss_mib":592.70703125},{"t_s":45,"rss_mib":592.70703125},{"t_s":45,"rss_mib":592.70703125},{"t_s":46,"rss_mib":592.70703125},{"t_s":46,"rss_mib":592.70703125},{"t_s":47,"rss_mib":592.70703125},{"t_s":47,"rss_mib":592.70703125},{"t_s":48,"rss_mib":592.70703125},{"t_s":49,"rss_mib":592.70703125},{"t_s":49,"rss_mib":592.70703125},{"t_s":50,"rss_mib":592.70703125},{"t_s":50,"rss_mib":592.70703125},{"t_s":50,"rss_mib":592.70703125},{"t_s":51,"rss_mib":592.70703125},{"t_s":51,"rss_mib":592.70703125},{"t_s":52,"rss_mib":592.70703125},{"t_s":52,"rss_mib":592.70703125},{"t_s":53,"rss_mib":592.70703125},{"t_s":54,"rss_mib":592.70703125},{"t_s":54,"rss_mib":592.70703125},{"t_s":55,"rss_mib":592.70703125},{"t_s":55,"rss_mib":592.70703125},{"t_s":55,"rss_mib":592.70703125},{"t_s":56,"rss_mib":592.70703125},{"t_s":56,"rss_mib":592.70703125},{"t_s":57,"rss_mib":592.70703125},{"t_s":57,"rss_mib":592.70703125},{"t_s":58,"rss_mib":592.70703125},{"t_s":59,"rss_mib":592.70703125},{"t_s":59,"rss_mib":592.70703125},{"t_s":59,"rss_mib":592.70703125}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.015894136,"cost":6.082916892,"memory":439.51128477,"streaming":0.000001781,"streams_sustained":0.000001616,"throughput":517.800213091},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/command-r:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":13881,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":14770,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":14798,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":14342.723004694835,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":69.72176759410802,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2507360988510164,"certified":true,"suppressed":false},"cost_window_ok":{"value":426,"certified":true,"suppressed":false},"cost_window_rps":{"value":70,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.06338028169014084,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":71,"certified":true,"suppressed":false},"concurrency":1,"p99_us":14828,"first_disqualified_conc":4,"lower_bound":false,"samples":427},{"bound_ms":100,"rps":{"value":71,"certified":true,"suppressed":false},"concurrency":1,"p99_us":14828,"first_disqualified_conc":8,"lower_bound":false,"samples":427},{"bound_ms":null,"rps":{"value":73,"certified":true,"suppressed":false},"concurrency":256,"p99_us":3307564,"first_disqualified_conc":8192,"lower_bound":false,"samples":439}],"sweep":[{"conc":1,"ok":427,"rps":71,"p99_us":14828,"fail":0},{"conc":1,"ok":425,"rps":70,"p99_us":14804,"fail":0},{"conc":1,"ok":425,"rps":70,"p99_us":14829,"fail":0},{"conc":2,"ok":428,"rps":71,"p99_us":28892,"fail":0},{"conc":2,"ok":428,"rps":71,"p99_us":28658,"fail":0},{"conc":2,"ok":430,"rps":71,"p99_us":29185,"fail":0},{"conc":4,"ok":432,"rps":71,"p99_us":60243,"fail":0},{"conc":4,"ok":430,"rps":71,"p99_us":60194,"fail":0},{"conc":4,"ok":432,"rps":71,"p99_us":59574,"fail":0},{"conc":8,"ok":430,"rps":71,"p99_us":121085,"fail":0},{"conc":8,"ok":429,"rps":71,"p99_us":122201,"fail":0},{"conc":8,"ok":429,"rps":71,"p99_us":120670,"fail":0},{"conc":16,"ok":426,"rps":70,"p99_us":247503,"fail":0},{"conc":16,"ok":427,"rps":71,"p99_us":244601,"fail":0},{"conc":16,"ok":425,"rps":70,"p99_us":254780,"fail":0},{"conc":32,"ok":424,"rps":70,"p99_us":680249,"fail":0},{"conc":32,"ok":425,"rps":70,"p99_us":712991,"fail":0},{"conc":32,"ok":425,"rps":70,"p99_us":694074,"fail":0},{"conc":64,"ok":387,"rps":64,"p99_us":2298977,"fail":0},{"conc":64,"ok":415,"rps":69,"p99_us":2262103,"fail":0},{"conc":64,"ok":418,"rps":69,"p99_us":2327408,"fail":0},{"conc":128,"ok":398,"rps":66,"p99_us":5538437,"fail":0},{"conc":128,"ok":384,"rps":63,"p99_us":1947524,"fail":0},{"conc":128,"ok":388,"rps":64,"p99_us":5662894,"fail":0},{"conc":256,"ok":368,"rps":61,"p99_us":5539552,"fail":0},{"conc":256,"ok":439,"rps":73,"p99_us":3307564,"fail":0},{"conc":256,"ok":337,"rps":56,"p99_us":5656871,"fail":0},{"conc":512,"ok":331,"rps":55,"p99_us":5332982,"fail":0},{"conc":512,"ok":398,"rps":66,"p99_us":5713851,"fail":0},{"conc":512,"ok":326,"rps":54,"p99_us":5051062,"fail":0},{"conc":1024,"ok":393,"rps":65,"p99_us":5585018,"fail":0},{"conc":1024,"ok":362,"rps":60,"p99_us":5010510,"fail":0},{"conc":1024,"ok":406,"rps":67,"p99_us":5630354,"fail":0},{"conc":2048,"ok":391,"rps":65,"p99_us":5594770,"fail":0},{"conc":2048,"ok":81,"rps":13,"p99_us":5939140,"fail":0},{"conc":2048,"ok":154,"rps":25,"p99_us":5880410,"fail":0},{"conc":4096,"ok":115,"rps":19,"p99_us":5753857,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":70,"certified":true,"suppressed":false},"p99_us":14828,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":28892,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":60194,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":121085,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":70,"certified":true,"suppressed":false},"p99_us":247503,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":70,"certified":true,"suppressed":false},"p99_us":694074,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":69,"certified":true,"suppressed":false},"p99_us":2298977,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":64,"certified":true,"suppressed":false},"p99_us":5538437,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":5539552,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":5332982,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":65,"certified":true,"suppressed":false},"p99_us":5585018,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":25,"certified":true,"suppressed":false},"p99_us":5880410,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":19,"certified":true,"suppressed":false},"p99_us":5753857,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 0 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 428 successful gateway round trip(s) and 262194 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":428,"direct_c1_samples":262194},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":593.359375,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":670.828125,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":670.9921875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":670.9921875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":670.9921875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.09640675558624215,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":593.359375},{"t_s":2,"rss_mib":665.65234375},{"t_s":2,"rss_mib":665.65234375},{"t_s":5,"rss_mib":665.875},{"t_s":6,"rss_mib":665.875},{"t_s":8,"rss_mib":666.17578125},{"t_s":8,"rss_mib":666.18359375},{"t_s":11,"rss_mib":666.765625},{"t_s":12,"rss_mib":666.76953125},{"t_s":14,"rss_mib":666.83203125},{"t_s":15,"rss_mib":666.8828125},{"t_s":17,"rss_mib":667.62890625},{"t_s":18,"rss_mib":667.68359375},{"t_s":20,"rss_mib":667.71875},{"t_s":21,"rss_mib":667.71875},{"t_s":23,"rss_mib":667.99609375},{"t_s":24,"rss_mib":667.99609375},{"t_s":24,"rss_mib":668.01171875},{"t_s":27,"rss_mib":668.01171875},{"t_s":30,"rss_mib":668.3359375},{"t_s":30,"rss_mib":668.3359375},{"t_s":33,"rss_mib":668.5234375},{"t_s":33,"rss_mib":668.5234375},{"t_s":35,"rss_mib":668.87890625},{"t_s":36,"rss_mib":668.87890625},{"t_s":37,"rss_mib":669.0234375},{"t_s":39,"rss_mib":669.0234375},{"t_s":42,"rss_mib":669.0234375},{"t_s":45,"rss_mib":669.0234375},{"t_s":48,"rss_mib":669.0234375},{"t_s":51,"rss_mib":669.0234375},{"t_s":54,"rss_mib":669.0234375},{"t_s":57,"rss_mib":669.0234375},{"t_s":60,"rss_mib":669.0234375},{"t_s":63,"rss_mib":669.0234375},{"t_s":66,"rss_mib":669.0234375},{"t_s":68,"rss_mib":669.046875},{"t_s":69,"rss_mib":669.046875},{"t_s":72,"rss_mib":669.0625},{"t_s":72,"rss_mib":669.08203125},{"t_s":75,"rss_mib":669.28515625},{"t_s":75,"rss_mib":669.28515625},{"t_s":77,"rss_mib":669.5625},{"t_s":78,"rss_mib":669.5625},{"t_s":80,"rss_mib":670.125},{"t_s":81,"rss_mib":670.125},{"t_s":83,"rss_mib":670.1796875},{"t_s":84,"rss_mib":670.1796875},{"t_s":86,"rss_mib":670.42578125},{"t_s":87,"rss_mib":670.4296875},{"t_s":90,"rss_mib":670.66796875},{"t_s":90,"rss_mib":670.66796875},{"t_s":93,"rss_mib":670.66796875},{"t_s":96,"rss_mib":670.66796875},{"t_s":99,"rss_mib":670.66796875},{"t_s":102,"rss_mib":670.66796875},{"t_s":105,"rss_mib":670.66796875},{"t_s":108,"rss_mib":670.66796875},{"t_s":111,"rss_mib":670.66796875},{"t_s":114,"rss_mib":670.66796875},{"t_s":117,"rss_mib":670.66796875},{"t_s":120,"rss_mib":670.66796875},{"t_s":123,"rss_mib":670.66796875},{"t_s":126,"rss_mib":670.66796875},{"t_s":128,"rss_mib":670.6796875},{"t_s":129,"rss_mib":670.6796875},{"t_s":132,"rss_mib":670.6796875},{"t_s":135,"rss_mib":670.6796875},{"t_s":138,"rss_mib":670.6796875},{"t_s":141,"rss_mib":670.6796875},{"t_s":143,"rss_mib":670.71484375},{"t_s":144,"rss_mib":670.71484375},{"t_s":147,"rss_mib":670.71484375},{"t_s":150,"rss_mib":670.71484375},{"t_s":153,"rss_mib":670.71484375},{"t_s":156,"rss_mib":670.71484375},{"t_s":159,"rss_mib":670.71484375},{"t_s":162,"rss_mib":670.71484375},{"t_s":165,"rss_mib":670.71484375},{"t_s":168,"rss_mib":670.71484375},{"t_s":172,"rss_mib":670.71484375},{"t_s":174,"rss_mib":670.71484375},{"t_s":178,"rss_mib":670.71484375},{"t_s":178,"rss_mib":670.71875},{"t_s":181,"rss_mib":670.71875},{"t_s":184,"rss_mib":670.71875},{"t_s":184,"rss_mib":670.72265625},{"t_s":187,"rss_mib":670.72265625},{"t_s":190,"rss_mib":670.72265625},{"t_s":193,"rss_mib":670.72265625},{"t_s":196,"rss_mib":670.72265625},{"t_s":198,"rss_mib":670.828125},{"t_s":199,"rss_mib":670.828125},{"t_s":202,"rss_mib":670.828125},{"t_s":205,"rss_mib":670.828125},{"t_s":208,"rss_mib":670.828125},{"t_s":211,"rss_mib":670.828125},{"t_s":214,"rss_mib":670.828125},{"t_s":217,"rss_mib":670.828125},{"t_s":220,"rss_mib":670.828125},{"t_s":223,"rss_mib":670.828125},{"t_s":226,"rss_mib":670.828125},{"t_s":229,"rss_mib":670.828125},{"t_s":232,"rss_mib":670.828125},{"t_s":235,"rss_mib":670.8359375},{"t_s":235,"rss_mib":670.8359375},{"t_s":238,"rss_mib":670.83984375},{"t_s":240,"rss_mib":670.84375},{"t_s":241,"rss_mib":670.84375},{"t_s":244,"rss_mib":670.8671875},{"t_s":244,"rss_mib":670.8671875},{"t_s":247,"rss_mib":670.8671875},{"t_s":249,"rss_mib":670.90625},{"t_s":250,"rss_mib":670.90625},{"t_s":251,"rss_mib":670.9921875},{"t_s":253,"rss_mib":670.9921875},{"t_s":256,"rss_mib":670.9921875},{"t_s":259,"rss_mib":670.9921875},{"t_s":262,"rss_mib":670.9921875},{"t_s":265,"rss_mib":670.9921875},{"t_s":268,"rss_mib":670.9921875},{"t_s":271,"rss_mib":670.9921875},{"t_s":274,"rss_mib":670.9921875},{"t_s":277,"rss_mib":670.9921875},{"t_s":280,"rss_mib":670.9921875},{"t_s":283,"rss_mib":670.9921875},{"t_s":286,"rss_mib":670.9921875},{"t_s":289,"rss_mib":670.9921875},{"t_s":292,"rss_mib":670.9921875},{"t_s":295,"rss_mib":670.9921875},{"t_s":298,"rss_mib":670.9921875},{"t_s":301,"rss_mib":670.9921875},{"t_s":304,"rss_mib":670.9921875},{"t_s":307,"rss_mib":670.9921875},{"t_s":310,"rss_mib":670.9921875},{"t_s":313,"rss_mib":670.9921875},{"t_s":316,"rss_mib":670.9921875},{"t_s":319,"rss_mib":670.9921875},{"t_s":322,"rss_mib":670.9921875},{"t_s":325,"rss_mib":670.9921875},{"t_s":328,"rss_mib":670.9921875},{"t_s":331,"rss_mib":670.9921875},{"t_s":334,"rss_mib":670.9921875},{"t_s":337,"rss_mib":670.9921875},{"t_s":341,"rss_mib":670.9921875},{"t_s":343,"rss_mib":670.9921875},{"t_s":347,"rss_mib":670.9921875},{"t_s":350,"rss_mib":670.9921875},{"t_s":353,"rss_mib":670.9921875},{"t_s":356,"rss_mib":670.9921875},{"t_s":359,"rss_mib":670.9921875},{"t_s":362,"rss_mib":670.9921875}],"idle_rss_series":[{"t_s":0,"rss_mib":593.359375},{"t_s":0,"rss_mib":593.359375},{"t_s":0,"rss_mib":593.359375},{"t_s":1,"rss_mib":593.359375},{"t_s":1,"rss_mib":593.359375},{"t_s":2,"rss_mib":593.359375},{"t_s":2,"rss_mib":593.359375},{"t_s":3,"rss_mib":593.359375},{"t_s":3,"rss_mib":593.359375},{"t_s":4,"rss_mib":593.359375},{"t_s":5,"rss_mib":593.359375},{"t_s":5,"rss_mib":593.359375},{"t_s":5,"rss_mib":593.359375},{"t_s":6,"rss_mib":593.359375},{"t_s":6,"rss_mib":593.359375},{"t_s":7,"rss_mib":593.359375},{"t_s":7,"rss_mib":593.359375},{"t_s":8,"rss_mib":593.359375},{"t_s":8,"rss_mib":593.359375},{"t_s":9,"rss_mib":593.359375},{"t_s":10,"rss_mib":593.359375},{"t_s":10,"rss_mib":593.359375},{"t_s":10,"rss_mib":593.359375},{"t_s":11,"rss_mib":593.359375},{"t_s":11,"rss_mib":593.359375},{"t_s":12,"rss_mib":593.359375},{"t_s":12,"rss_mib":593.359375},{"t_s":13,"rss_mib":593.359375},{"t_s":13,"rss_mib":593.359375},{"t_s":14,"rss_mib":593.359375},{"t_s":15,"rss_mib":593.359375},{"t_s":15,"rss_mib":593.359375},{"t_s":15,"rss_mib":593.359375},{"t_s":16,"rss_mib":593.359375},{"t_s":16,"rss_mib":593.359375},{"t_s":17,"rss_mib":593.359375},{"t_s":17,"rss_mib":593.359375},{"t_s":18,"rss_mib":593.359375},{"t_s":18,"rss_mib":593.359375},{"t_s":19,"rss_mib":593.359375},{"t_s":20,"rss_mib":593.359375},{"t_s":20,"rss_mib":593.359375},{"t_s":20,"rss_mib":593.359375},{"t_s":21,"rss_mib":593.359375},{"t_s":21,"rss_mib":593.359375},{"t_s":22,"rss_mib":593.359375},{"t_s":22,"rss_mib":593.359375},{"t_s":23,"rss_mib":593.359375},{"t_s":23,"rss_mib":593.359375},{"t_s":24,"rss_mib":593.359375},{"t_s":25,"rss_mib":593.359375},{"t_s":25,"rss_mib":593.359375},{"t_s":25,"rss_mib":593.359375},{"t_s":26,"rss_mib":593.359375},{"t_s":26,"rss_mib":593.359375},{"t_s":27,"rss_mib":593.359375},{"t_s":27,"rss_mib":593.359375},{"t_s":28,"rss_mib":593.359375},{"t_s":28,"rss_mib":593.359375},{"t_s":29,"rss_mib":593.359375},{"t_s":30,"rss_mib":593.359375},{"t_s":30,"rss_mib":593.359375},{"t_s":30,"rss_mib":593.359375},{"t_s":31,"rss_mib":593.359375},{"t_s":31,"rss_mib":593.359375},{"t_s":32,"rss_mib":593.359375},{"t_s":32,"rss_mib":593.359375},{"t_s":33,"rss_mib":593.359375},{"t_s":33,"rss_mib":593.359375},{"t_s":34,"rss_mib":593.359375},{"t_s":35,"rss_mib":593.359375},{"t_s":35,"rss_mib":593.359375},{"t_s":35,"rss_mib":593.359375},{"t_s":36,"rss_mib":593.359375},{"t_s":36,"rss_mib":593.359375},{"t_s":37,"rss_mib":593.359375},{"t_s":37,"rss_mib":593.359375},{"t_s":38,"rss_mib":593.359375},{"t_s":38,"rss_mib":593.359375},{"t_s":39,"rss_mib":593.359375},{"t_s":40,"rss_mib":593.359375},{"t_s":40,"rss_mib":593.359375},{"t_s":40,"rss_mib":593.359375},{"t_s":41,"rss_mib":593.359375},{"t_s":41,"rss_mib":593.359375},{"t_s":42,"rss_mib":593.359375},{"t_s":42,"rss_mib":593.359375},{"t_s":43,"rss_mib":593.359375},{"t_s":43,"rss_mib":593.359375},{"t_s":44,"rss_mib":593.359375},{"t_s":45,"rss_mib":593.359375},{"t_s":45,"rss_mib":593.359375},{"t_s":45,"rss_mib":593.359375},{"t_s":46,"rss_mib":593.359375},{"t_s":46,"rss_mib":593.359375},{"t_s":47,"rss_mib":593.359375},{"t_s":47,"rss_mib":593.359375},{"t_s":48,"rss_mib":593.359375},{"t_s":48,"rss_mib":593.359375},{"t_s":49,"rss_mib":593.359375},{"t_s":50,"rss_mib":593.359375},{"t_s":50,"rss_mib":593.359375},{"t_s":50,"rss_mib":593.359375},{"t_s":51,"rss_mib":593.359375},{"t_s":51,"rss_mib":593.359375},{"t_s":52,"rss_mib":593.359375},{"t_s":52,"rss_mib":593.359375},{"t_s":53,"rss_mib":593.359375},{"t_s":53,"rss_mib":593.359375},{"t_s":54,"rss_mib":593.359375},{"t_s":55,"rss_mib":593.359375},{"t_s":55,"rss_mib":593.359375},{"t_s":55,"rss_mib":593.359375},{"t_s":56,"rss_mib":593.359375},{"t_s":56,"rss_mib":593.359375},{"t_s":57,"rss_mib":593.359375},{"t_s":57,"rss_mib":593.359375},{"t_s":58,"rss_mib":593.359375},{"t_s":58,"rss_mib":593.359375},{"t_s":59,"rss_mib":593.359375},{"t_s":59,"rss_mib":593.359375}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.017493439,"cost":6.105363014,"memory":437.62820347,"streaming":0.000001541,"streams_sustained":0.000001915,"throughput":563.04593918},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}}}},"anthropic":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"cohere":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/v2/chat","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"bedrock":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/model/gpt-4o-mini-anthropic/converse","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":13061,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":13826,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":13854,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":13414.096916299559,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":74.54844006568145,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25025039298228663,"certified":true,"suppressed":false},"cost_window_ok":{"value":454,"certified":true,"suppressed":false},"cost_window_rps":{"value":75,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.05506607929515418,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":76,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13878,"first_disqualified_conc":4,"lower_bound":false,"samples":458},{"bound_ms":100,"rps":{"value":76,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13878,"first_disqualified_conc":8,"lower_bound":false,"samples":458},{"bound_ms":null,"rps":{"value":76,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13878,"first_disqualified_conc":8192,"lower_bound":false,"samples":458}],"sweep":[{"conc":1,"ok":461,"rps":76,"p99_us":13653,"fail":0},{"conc":1,"ok":460,"rps":76,"p99_us":13644,"fail":0},{"conc":1,"ok":458,"rps":76,"p99_us":13878,"fail":0},{"conc":2,"ok":460,"rps":76,"p99_us":27434,"fail":0},{"conc":2,"ok":462,"rps":76,"p99_us":27476,"fail":0},{"conc":2,"ok":462,"rps":76,"p99_us":27403,"fail":0},{"conc":4,"ok":458,"rps":76,"p99_us":55279,"fail":0},{"conc":4,"ok":461,"rps":76,"p99_us":52761,"fail":0},{"conc":4,"ok":460,"rps":76,"p99_us":55178,"fail":0},{"conc":8,"ok":457,"rps":76,"p99_us":113676,"fail":0},{"conc":8,"ok":457,"rps":76,"p99_us":113508,"fail":0},{"conc":8,"ok":458,"rps":76,"p99_us":111569,"fail":0},{"conc":16,"ok":457,"rps":76,"p99_us":225192,"fail":0},{"conc":16,"ok":457,"rps":76,"p99_us":226101,"fail":0},{"conc":16,"ok":457,"rps":76,"p99_us":225145,"fail":0},{"conc":32,"ok":460,"rps":76,"p99_us":534376,"fail":0},{"conc":32,"ok":453,"rps":75,"p99_us":604109,"fail":0},{"conc":32,"ok":454,"rps":75,"p99_us":611626,"fail":0},{"conc":64,"ok":450,"rps":74,"p99_us":2090299,"fail":0},{"conc":64,"ok":451,"rps":75,"p99_us":899777,"fail":0},{"conc":64,"ok":439,"rps":73,"p99_us":1369331,"fail":0},{"conc":128,"ok":428,"rps":71,"p99_us":5452146,"fail":0},{"conc":128,"ok":384,"rps":63,"p99_us":2310017,"fail":0},{"conc":128,"ok":428,"rps":71,"p99_us":5510520,"fail":0},{"conc":256,"ok":340,"rps":56,"p99_us":3730579,"fail":0},{"conc":256,"ok":438,"rps":72,"p99_us":5472992,"fail":0},{"conc":256,"ok":287,"rps":47,"p99_us":3548169,"fail":0},{"conc":512,"ok":422,"rps":70,"p99_us":5595291,"fail":0},{"conc":512,"ok":398,"rps":66,"p99_us":5632383,"fail":0},{"conc":512,"ok":321,"rps":53,"p99_us":5271242,"fail":0},{"conc":1024,"ok":362,"rps":60,"p99_us":4574004,"fail":0},{"conc":1024,"ok":360,"rps":59,"p99_us":5037455,"fail":0},{"conc":1024,"ok":354,"rps":57,"p99_us":5052953,"fail":0},{"conc":2048,"ok":365,"rps":60,"p99_us":4970886,"fail":0},{"conc":2048,"ok":298,"rps":49,"p99_us":5830717,"fail":0},{"conc":2048,"ok":280,"rps":46,"p99_us":5758280,"fail":0},{"conc":4096,"ok":320,"rps":52,"p99_us":5620986,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":954},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":492},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":76,"certified":true,"suppressed":false},"p99_us":13653,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":76,"certified":true,"suppressed":false},"p99_us":27434,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":76,"certified":true,"suppressed":false},"p99_us":55178,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":76,"certified":true,"suppressed":false},"p99_us":113508,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":76,"certified":true,"suppressed":false},"p99_us":225192,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":75,"certified":true,"suppressed":false},"p99_us":604109,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":1369331,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":5452146,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":3730579,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":66,"certified":true,"suppressed":false},"p99_us":5595291,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":5037455,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":49,"certified":true,"suppressed":false},"p99_us":5758280,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":52,"certified":true,"suppressed":false},"p99_us":5620986,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 457 successful gateway round trip(s) and 263938 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":457,"direct_c1_samples":263938},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.65625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":667.640625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":667.640625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":667.640625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":667.640625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.65625},{"t_s":1,"rss_mib":667.640625},{"t_s":2,"rss_mib":667.640625},{"t_s":6,"rss_mib":667.640625},{"t_s":8,"rss_mib":667.640625},{"t_s":12,"rss_mib":667.640625},{"t_s":15,"rss_mib":667.640625},{"t_s":18,"rss_mib":667.640625},{"t_s":21,"rss_mib":667.640625},{"t_s":24,"rss_mib":667.640625},{"t_s":27,"rss_mib":667.640625},{"t_s":30,"rss_mib":667.640625},{"t_s":33,"rss_mib":667.640625},{"t_s":36,"rss_mib":667.640625},{"t_s":39,"rss_mib":667.640625},{"t_s":42,"rss_mib":667.640625},{"t_s":45,"rss_mib":667.640625},{"t_s":48,"rss_mib":667.640625},{"t_s":51,"rss_mib":667.640625},{"t_s":54,"rss_mib":667.640625},{"t_s":57,"rss_mib":667.640625},{"t_s":60,"rss_mib":667.640625},{"t_s":63,"rss_mib":667.640625},{"t_s":66,"rss_mib":667.640625},{"t_s":69,"rss_mib":667.640625},{"t_s":72,"rss_mib":667.640625},{"t_s":75,"rss_mib":667.640625},{"t_s":78,"rss_mib":667.640625},{"t_s":81,"rss_mib":667.640625},{"t_s":84,"rss_mib":667.640625},{"t_s":87,"rss_mib":667.640625},{"t_s":90,"rss_mib":667.640625},{"t_s":93,"rss_mib":667.640625},{"t_s":96,"rss_mib":667.640625},{"t_s":99,"rss_mib":667.640625},{"t_s":102,"rss_mib":667.640625},{"t_s":105,"rss_mib":667.640625},{"t_s":108,"rss_mib":667.640625},{"t_s":111,"rss_mib":667.640625},{"t_s":114,"rss_mib":667.640625},{"t_s":117,"rss_mib":667.640625},{"t_s":120,"rss_mib":667.640625},{"t_s":123,"rss_mib":667.640625},{"t_s":126,"rss_mib":667.640625},{"t_s":129,"rss_mib":667.640625},{"t_s":132,"rss_mib":667.640625},{"t_s":135,"rss_mib":667.640625},{"t_s":138,"rss_mib":667.640625},{"t_s":141,"rss_mib":667.640625},{"t_s":144,"rss_mib":667.640625},{"t_s":147,"rss_mib":667.640625},{"t_s":150,"rss_mib":667.640625},{"t_s":153,"rss_mib":667.640625},{"t_s":156,"rss_mib":667.640625},{"t_s":159,"rss_mib":667.640625},{"t_s":162,"rss_mib":667.640625},{"t_s":165,"rss_mib":667.640625},{"t_s":169,"rss_mib":667.640625},{"t_s":171,"rss_mib":667.640625},{"t_s":175,"rss_mib":667.640625},{"t_s":178,"rss_mib":667.640625},{"t_s":181,"rss_mib":667.640625},{"t_s":184,"rss_mib":667.640625},{"t_s":187,"rss_mib":667.640625},{"t_s":190,"rss_mib":667.640625},{"t_s":193,"rss_mib":667.640625},{"t_s":196,"rss_mib":667.640625},{"t_s":199,"rss_mib":667.640625},{"t_s":202,"rss_mib":667.640625},{"t_s":205,"rss_mib":667.640625},{"t_s":208,"rss_mib":667.640625},{"t_s":211,"rss_mib":667.640625},{"t_s":214,"rss_mib":667.640625},{"t_s":217,"rss_mib":667.640625},{"t_s":220,"rss_mib":667.640625},{"t_s":223,"rss_mib":667.640625},{"t_s":226,"rss_mib":667.640625},{"t_s":229,"rss_mib":667.640625},{"t_s":232,"rss_mib":667.640625},{"t_s":235,"rss_mib":667.640625},{"t_s":238,"rss_mib":667.640625},{"t_s":241,"rss_mib":667.640625},{"t_s":244,"rss_mib":667.640625},{"t_s":247,"rss_mib":667.640625},{"t_s":250,"rss_mib":667.640625},{"t_s":253,"rss_mib":667.640625},{"t_s":256,"rss_mib":667.640625},{"t_s":259,"rss_mib":667.640625},{"t_s":262,"rss_mib":667.640625},{"t_s":265,"rss_mib":667.640625},{"t_s":268,"rss_mib":667.640625},{"t_s":271,"rss_mib":667.640625},{"t_s":274,"rss_mib":667.640625},{"t_s":277,"rss_mib":667.640625},{"t_s":280,"rss_mib":667.640625},{"t_s":283,"rss_mib":667.640625},{"t_s":286,"rss_mib":667.640625},{"t_s":289,"rss_mib":667.640625},{"t_s":292,"rss_mib":667.640625},{"t_s":295,"rss_mib":667.640625},{"t_s":298,"rss_mib":667.640625},{"t_s":301,"rss_mib":667.640625},{"t_s":304,"rss_mib":667.640625},{"t_s":307,"rss_mib":667.640625},{"t_s":310,"rss_mib":667.640625},{"t_s":313,"rss_mib":667.640625},{"t_s":316,"rss_mib":667.640625},{"t_s":319,"rss_mib":667.640625},{"t_s":322,"rss_mib":667.640625},{"t_s":325,"rss_mib":667.640625},{"t_s":328,"rss_mib":667.640625},{"t_s":331,"rss_mib":667.640625},{"t_s":335,"rss_mib":667.640625},{"t_s":337,"rss_mib":667.640625},{"t_s":341,"rss_mib":667.640625},{"t_s":344,"rss_mib":667.640625},{"t_s":347,"rss_mib":667.640625},{"t_s":350,"rss_mib":667.640625},{"t_s":353,"rss_mib":667.640625},{"t_s":356,"rss_mib":667.640625},{"t_s":359,"rss_mib":667.640625},{"t_s":362,"rss_mib":667.640625}],"idle_rss_series":[{"t_s":0,"rss_mib":592.65625},{"t_s":0,"rss_mib":592.65625},{"t_s":0,"rss_mib":592.65625},{"t_s":1,"rss_mib":592.65625},{"t_s":1,"rss_mib":592.65625},{"t_s":2,"rss_mib":592.65625},{"t_s":2,"rss_mib":592.65625},{"t_s":3,"rss_mib":592.65625},{"t_s":3,"rss_mib":592.65625},{"t_s":4,"rss_mib":592.65625},{"t_s":5,"rss_mib":592.65625},{"t_s":5,"rss_mib":592.65625},{"t_s":5,"rss_mib":592.65625},{"t_s":6,"rss_mib":592.65625},{"t_s":6,"rss_mib":592.65625},{"t_s":7,"rss_mib":592.65625},{"t_s":7,"rss_mib":592.65625},{"t_s":8,"rss_mib":592.65625},{"t_s":8,"rss_mib":592.65625},{"t_s":9,"rss_mib":592.65625},{"t_s":10,"rss_mib":592.65625},{"t_s":10,"rss_mib":592.65625},{"t_s":10,"rss_mib":592.65625},{"t_s":11,"rss_mib":592.65625},{"t_s":11,"rss_mib":592.65625},{"t_s":12,"rss_mib":592.65625},{"t_s":12,"rss_mib":592.65625},{"t_s":13,"rss_mib":592.65625},{"t_s":13,"rss_mib":592.65625},{"t_s":14,"rss_mib":592.65625},{"t_s":15,"rss_mib":592.65625},{"t_s":15,"rss_mib":592.65625},{"t_s":15,"rss_mib":592.65625},{"t_s":16,"rss_mib":592.65625},{"t_s":16,"rss_mib":592.65625},{"t_s":17,"rss_mib":592.65625},{"t_s":17,"rss_mib":592.65625},{"t_s":18,"rss_mib":592.65625},{"t_s":18,"rss_mib":592.65625},{"t_s":19,"rss_mib":592.65625},{"t_s":20,"rss_mib":592.65625},{"t_s":20,"rss_mib":592.65625},{"t_s":20,"rss_mib":592.65625},{"t_s":21,"rss_mib":592.65625},{"t_s":21,"rss_mib":592.65625},{"t_s":22,"rss_mib":592.65625},{"t_s":22,"rss_mib":592.65625},{"t_s":23,"rss_mib":592.65625},{"t_s":23,"rss_mib":592.65625},{"t_s":24,"rss_mib":592.65625},{"t_s":25,"rss_mib":592.65625},{"t_s":25,"rss_mib":592.65625},{"t_s":25,"rss_mib":592.65625},{"t_s":26,"rss_mib":592.65625},{"t_s":26,"rss_mib":592.65625},{"t_s":27,"rss_mib":592.65625},{"t_s":27,"rss_mib":592.65625},{"t_s":28,"rss_mib":592.65625},{"t_s":28,"rss_mib":592.65625},{"t_s":29,"rss_mib":592.65625},{"t_s":30,"rss_mib":592.65625},{"t_s":30,"rss_mib":592.65625},{"t_s":30,"rss_mib":592.65625},{"t_s":31,"rss_mib":592.65625},{"t_s":31,"rss_mib":592.65625},{"t_s":32,"rss_mib":592.65625},{"t_s":32,"rss_mib":592.65625},{"t_s":33,"rss_mib":592.65625},{"t_s":33,"rss_mib":592.65625},{"t_s":34,"rss_mib":592.65625},{"t_s":35,"rss_mib":592.65625},{"t_s":35,"rss_mib":592.65625},{"t_s":35,"rss_mib":592.65625},{"t_s":36,"rss_mib":592.65625},{"t_s":36,"rss_mib":592.65625},{"t_s":37,"rss_mib":592.65625},{"t_s":37,"rss_mib":592.65625},{"t_s":38,"rss_mib":592.65625},{"t_s":38,"rss_mib":592.65625},{"t_s":39,"rss_mib":592.65625},{"t_s":40,"rss_mib":592.65625},{"t_s":40,"rss_mib":592.65625},{"t_s":40,"rss_mib":592.65625},{"t_s":41,"rss_mib":592.65625},{"t_s":41,"rss_mib":592.65625},{"t_s":42,"rss_mib":592.65625},{"t_s":42,"rss_mib":592.65625},{"t_s":43,"rss_mib":592.65625},{"t_s":44,"rss_mib":592.65625},{"t_s":44,"rss_mib":592.65625},{"t_s":45,"rss_mib":592.65625},{"t_s":45,"rss_mib":592.65625},{"t_s":45,"rss_mib":592.65625},{"t_s":46,"rss_mib":592.65625},{"t_s":46,"rss_mib":592.65625},{"t_s":47,"rss_mib":592.65625},{"t_s":47,"rss_mib":592.65625},{"t_s":48,"rss_mib":592.65625},{"t_s":49,"rss_mib":592.65625},{"t_s":49,"rss_mib":592.65625},{"t_s":50,"rss_mib":592.65625},{"t_s":50,"rss_mib":592.65625},{"t_s":50,"rss_mib":592.65625},{"t_s":51,"rss_mib":592.65625},{"t_s":51,"rss_mib":592.65625},{"t_s":52,"rss_mib":592.65625},{"t_s":52,"rss_mib":592.65625},{"t_s":53,"rss_mib":592.65625},{"t_s":54,"rss_mib":592.65625},{"t_s":54,"rss_mib":592.65625},{"t_s":55,"rss_mib":592.65625},{"t_s":55,"rss_mib":592.65625},{"t_s":55,"rss_mib":592.65625},{"t_s":56,"rss_mib":592.65625},{"t_s":56,"rss_mib":592.65625},{"t_s":57,"rss_mib":592.65625},{"t_s":57,"rss_mib":592.65625},{"t_s":58,"rss_mib":592.65625},{"t_s":59,"rss_mib":592.65625},{"t_s":59,"rss_mib":592.65625},{"t_s":59,"rss_mib":592.65625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.010785541,"cost":6.097304632,"memory":437.619000209,"streaming":0.000001741,"streams_sustained":0.000001469,"throughput":558.563165961},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/gpt-4o-mini-anthropic:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":16324,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":17226,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":17254,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":16831.955922865014,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":59.41080196399345,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2501531621486448,"certified":true,"suppressed":false},"cost_window_ok":{"value":363,"certified":true,"suppressed":false},"cost_window_rps":{"value":60,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.0743801652892562,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":60,"certified":true,"suppressed":false},"concurrency":2,"p99_us":33932,"first_disqualified_conc":4,"lower_bound":false,"samples":362},{"bound_ms":100,"rps":{"value":60,"certified":true,"suppressed":false},"concurrency":2,"p99_us":33932,"first_disqualified_conc":8,"lower_bound":false,"samples":362},{"bound_ms":null,"rps":{"value":60,"certified":true,"suppressed":false},"concurrency":2,"p99_us":33932,"first_disqualified_conc":4096,"lower_bound":false,"samples":362}],"sweep":[{"conc":1,"ok":359,"rps":59,"p99_us":17544,"fail":0},{"conc":1,"ok":359,"rps":59,"p99_us":17507,"fail":0},{"conc":1,"ok":359,"rps":59,"p99_us":17521,"fail":0},{"conc":2,"ok":360,"rps":59,"p99_us":35565,"fail":0},{"conc":2,"ok":362,"rps":60,"p99_us":34100,"fail":0},{"conc":2,"ok":362,"rps":60,"p99_us":33932,"fail":0},{"conc":4,"ok":360,"rps":59,"p99_us":69623,"fail":0},{"conc":4,"ok":358,"rps":59,"p99_us":71381,"fail":0},{"conc":4,"ok":360,"rps":59,"p99_us":68862,"fail":0},{"conc":8,"ok":359,"rps":59,"p99_us":142465,"fail":0},{"conc":8,"ok":358,"rps":59,"p99_us":142177,"fail":0},{"conc":8,"ok":359,"rps":59,"p99_us":142557,"fail":0},{"conc":16,"ok":354,"rps":57,"p99_us":295915,"fail":0},{"conc":16,"ok":357,"rps":59,"p99_us":287929,"fail":0},{"conc":16,"ok":357,"rps":59,"p99_us":299841,"fail":0},{"conc":32,"ok":354,"rps":57,"p99_us":859335,"fail":0},{"conc":32,"ok":355,"rps":59,"p99_us":841581,"fail":0},{"conc":32,"ok":351,"rps":58,"p99_us":836100,"fail":0},{"conc":64,"ok":315,"rps":52,"p99_us":2779385,"fail":0},{"conc":64,"ok":344,"rps":57,"p99_us":2751531,"fail":0},{"conc":64,"ok":346,"rps":57,"p99_us":2787524,"fail":0},{"conc":128,"ok":333,"rps":55,"p99_us":5623286,"fail":0},{"conc":128,"ok":319,"rps":53,"p99_us":5394586,"fail":0},{"conc":128,"ok":331,"rps":55,"p99_us":5555557,"fail":0},{"conc":256,"ok":253,"rps":42,"p99_us":4827293,"fail":0},{"conc":256,"ok":316,"rps":52,"p99_us":5720071,"fail":0},{"conc":256,"ok":330,"rps":54,"p99_us":5662924,"fail":0},{"conc":512,"ok":336,"rps":55,"p99_us":5617332,"fail":0},{"conc":512,"ok":290,"rps":48,"p99_us":5240198,"fail":0},{"conc":512,"ok":306,"rps":50,"p99_us":4955674,"fail":0},{"conc":1024,"ok":352,"rps":58,"p99_us":5558937,"fail":0},{"conc":1024,"ok":334,"rps":55,"p99_us":5693361,"fail":0},{"conc":1024,"ok":328,"rps":54,"p99_us":5448339,"fail":0},{"conc":2048,"ok":307,"rps":51,"p99_us":5588989,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":17521,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":34100,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":69623,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":142465,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":295915,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":58,"certified":true,"suppressed":false},"p99_us":841581,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":57,"certified":true,"suppressed":false},"p99_us":2779385,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":5555557,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":52,"certified":true,"suppressed":false},"p99_us":5662924,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":50,"certified":true,"suppressed":false},"p99_us":5240198,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":5558937,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":51,"certified":true,"suppressed":false},"p99_us":5588989,"windows":3,"clean_windows":1},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 0 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 364 successful gateway round trip(s) and 262033 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":364,"direct_c1_samples":262033},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.91796875,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":671.87109375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":305,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.91796875},{"t_s":1,"rss_mib":665.58984375},{"t_s":2,"rss_mib":665.58984375},{"t_s":5,"rss_mib":665.8515625},{"t_s":6,"rss_mib":666.03125},{"t_s":8,"rss_mib":666.31640625},{"t_s":9,"rss_mib":666.31640625},{"t_s":11,"rss_mib":666.71484375},{"t_s":12,"rss_mib":666.71484375},{"t_s":14,"rss_mib":666.72265625},{"t_s":15,"rss_mib":666.72265625},{"t_s":17,"rss_mib":667.28515625},{"t_s":18,"rss_mib":667.28515625},{"t_s":20,"rss_mib":667.359375},{"t_s":21,"rss_mib":667.359375},{"t_s":24,"rss_mib":667.63671875},{"t_s":24,"rss_mib":667.66015625},{"t_s":27,"rss_mib":667.80078125},{"t_s":27,"rss_mib":667.86328125},{"t_s":29,"rss_mib":668.296875},{"t_s":30,"rss_mib":668.296875},{"t_s":33,"rss_mib":668.32421875},{"t_s":33,"rss_mib":668.33203125},{"t_s":35,"rss_mib":668.68359375},{"t_s":36,"rss_mib":668.68359375},{"t_s":38,"rss_mib":668.953125},{"t_s":39,"rss_mib":668.953125},{"t_s":40,"rss_mib":668.9921875},{"t_s":42,"rss_mib":668.9921875},{"t_s":45,"rss_mib":668.9921875},{"t_s":48,"rss_mib":668.9921875},{"t_s":51,"rss_mib":668.9921875},{"t_s":53,"rss_mib":669.00390625},{"t_s":54,"rss_mib":669.00390625},{"t_s":56,"rss_mib":669.01171875},{"t_s":57,"rss_mib":669.01171875},{"t_s":60,"rss_mib":669.01171875},{"t_s":63,"rss_mib":669.01171875},{"t_s":66,"rss_mib":669.3203125},{"t_s":66,"rss_mib":669.3203125},{"t_s":67,"rss_mib":669.60546875},{"t_s":69,"rss_mib":669.60546875},{"t_s":72,"rss_mib":669.875},{"t_s":73,"rss_mib":669.875},{"t_s":75,"rss_mib":670.06640625},{"t_s":76,"rss_mib":670.06640625},{"t_s":78,"rss_mib":670.4609375},{"t_s":79,"rss_mib":670.4609375},{"t_s":82,"rss_mib":670.77734375},{"t_s":82,"rss_mib":670.77734375},{"t_s":82,"rss_mib":670.78125},{"t_s":85,"rss_mib":670.796875},{"t_s":87,"rss_mib":671.17578125},{"t_s":88,"rss_mib":671.17578125},{"t_s":89,"rss_mib":671.1953125},{"t_s":91,"rss_mib":671.1953125},{"t_s":93,"rss_mib":671.46875},{"t_s":94,"rss_mib":671.46875},{"t_s":95,"rss_mib":671.49609375},{"t_s":97,"rss_mib":671.49609375},{"t_s":100,"rss_mib":671.84765625},{"t_s":100,"rss_mib":671.84765625},{"t_s":101,"rss_mib":671.8515625},{"t_s":103,"rss_mib":671.8515625},{"t_s":106,"rss_mib":671.8515625},{"t_s":109,"rss_mib":671.8515625},{"t_s":112,"rss_mib":671.8515625},{"t_s":115,"rss_mib":671.8515625},{"t_s":118,"rss_mib":671.8515625},{"t_s":121,"rss_mib":671.8515625},{"t_s":124,"rss_mib":671.8515625},{"t_s":127,"rss_mib":671.8515625},{"t_s":130,"rss_mib":671.8515625},{"t_s":133,"rss_mib":671.8515625},{"t_s":137,"rss_mib":671.8515625},{"t_s":139,"rss_mib":671.8515625},{"t_s":143,"rss_mib":671.8515625},{"t_s":146,"rss_mib":671.8515625},{"t_s":149,"rss_mib":671.8515625},{"t_s":151,"rss_mib":671.86328125},{"t_s":152,"rss_mib":671.86328125},{"t_s":155,"rss_mib":671.86328125},{"t_s":155,"rss_mib":671.8671875},{"t_s":158,"rss_mib":671.8671875},{"t_s":160,"rss_mib":671.87109375},{"t_s":161,"rss_mib":671.87109375},{"t_s":164,"rss_mib":671.87109375},{"t_s":167,"rss_mib":671.87109375},{"t_s":170,"rss_mib":671.87109375},{"t_s":173,"rss_mib":671.87109375},{"t_s":176,"rss_mib":671.87109375},{"t_s":179,"rss_mib":671.87109375},{"t_s":182,"rss_mib":671.87109375},{"t_s":185,"rss_mib":671.87109375},{"t_s":188,"rss_mib":671.87109375},{"t_s":191,"rss_mib":671.87109375},{"t_s":194,"rss_mib":671.87109375},{"t_s":197,"rss_mib":671.87109375},{"t_s":200,"rss_mib":671.87109375},{"t_s":203,"rss_mib":671.87109375},{"t_s":207,"rss_mib":671.87109375},{"t_s":210,"rss_mib":671.87109375},{"t_s":213,"rss_mib":671.87109375},{"t_s":216,"rss_mib":671.87109375},{"t_s":219,"rss_mib":671.87109375},{"t_s":222,"rss_mib":671.87109375},{"t_s":225,"rss_mib":671.87109375},{"t_s":228,"rss_mib":671.87109375},{"t_s":231,"rss_mib":671.87109375},{"t_s":234,"rss_mib":671.87109375},{"t_s":237,"rss_mib":671.87109375},{"t_s":240,"rss_mib":671.87109375},{"t_s":243,"rss_mib":671.87109375},{"t_s":246,"rss_mib":671.87109375},{"t_s":249,"rss_mib":671.87109375},{"t_s":252,"rss_mib":671.87109375},{"t_s":255,"rss_mib":671.87109375},{"t_s":258,"rss_mib":671.87109375},{"t_s":261,"rss_mib":671.87109375},{"t_s":264,"rss_mib":671.87109375},{"t_s":267,"rss_mib":671.87109375},{"t_s":270,"rss_mib":671.87109375},{"t_s":274,"rss_mib":671.87109375},{"t_s":276,"rss_mib":671.87109375},{"t_s":280,"rss_mib":671.87109375},{"t_s":283,"rss_mib":671.87109375},{"t_s":286,"rss_mib":671.87109375},{"t_s":289,"rss_mib":671.87109375},{"t_s":292,"rss_mib":671.87109375},{"t_s":295,"rss_mib":671.87109375},{"t_s":298,"rss_mib":671.87109375},{"t_s":301,"rss_mib":671.87109375},{"t_s":304,"rss_mib":671.87109375},{"t_s":307,"rss_mib":671.87109375},{"t_s":310,"rss_mib":671.87109375},{"t_s":313,"rss_mib":671.87109375},{"t_s":316,"rss_mib":671.87109375},{"t_s":319,"rss_mib":671.87109375},{"t_s":322,"rss_mib":671.87109375},{"t_s":325,"rss_mib":671.87109375},{"t_s":328,"rss_mib":671.87109375},{"t_s":331,"rss_mib":671.87109375},{"t_s":334,"rss_mib":671.87109375},{"t_s":337,"rss_mib":671.87109375},{"t_s":340,"rss_mib":671.87109375},{"t_s":343,"rss_mib":671.87109375},{"t_s":347,"rss_mib":671.87109375},{"t_s":350,"rss_mib":671.87109375},{"t_s":353,"rss_mib":671.87109375},{"t_s":356,"rss_mib":671.87109375},{"t_s":359,"rss_mib":671.87109375},{"t_s":362,"rss_mib":671.87109375},{"t_s":365,"rss_mib":671.87109375}],"idle_rss_series":[{"t_s":0,"rss_mib":591.91796875},{"t_s":0,"rss_mib":591.91796875},{"t_s":0,"rss_mib":591.91796875},{"t_s":1,"rss_mib":591.91796875},{"t_s":1,"rss_mib":591.91796875},{"t_s":2,"rss_mib":591.91796875},{"t_s":2,"rss_mib":591.91796875},{"t_s":3,"rss_mib":591.91796875},{"t_s":3,"rss_mib":591.91796875},{"t_s":4,"rss_mib":591.91796875},{"t_s":5,"rss_mib":591.91796875},{"t_s":5,"rss_mib":591.91796875},{"t_s":5,"rss_mib":591.91796875},{"t_s":6,"rss_mib":591.91796875},{"t_s":6,"rss_mib":591.91796875},{"t_s":7,"rss_mib":591.91796875},{"t_s":7,"rss_mib":591.91796875},{"t_s":8,"rss_mib":591.91796875},{"t_s":8,"rss_mib":591.91796875},{"t_s":9,"rss_mib":591.91796875},{"t_s":10,"rss_mib":591.91796875},{"t_s":10,"rss_mib":591.91796875},{"t_s":10,"rss_mib":591.91796875},{"t_s":11,"rss_mib":591.91796875},{"t_s":11,"rss_mib":591.91796875},{"t_s":12,"rss_mib":591.91796875},{"t_s":12,"rss_mib":591.91796875},{"t_s":13,"rss_mib":591.91796875},{"t_s":13,"rss_mib":591.91796875},{"t_s":14,"rss_mib":591.91796875},{"t_s":15,"rss_mib":591.91796875},{"t_s":15,"rss_mib":591.91796875},{"t_s":15,"rss_mib":591.91796875},{"t_s":16,"rss_mib":591.91796875},{"t_s":16,"rss_mib":591.91796875},{"t_s":17,"rss_mib":591.91796875},{"t_s":17,"rss_mib":591.91796875},{"t_s":18,"rss_mib":591.91796875},{"t_s":18,"rss_mib":591.91796875},{"t_s":19,"rss_mib":591.91796875},{"t_s":20,"rss_mib":591.91796875},{"t_s":20,"rss_mib":591.91796875},{"t_s":20,"rss_mib":591.91796875},{"t_s":21,"rss_mib":591.91796875},{"t_s":21,"rss_mib":591.91796875},{"t_s":22,"rss_mib":591.91796875},{"t_s":22,"rss_mib":591.91796875},{"t_s":23,"rss_mib":591.91796875},{"t_s":23,"rss_mib":591.91796875},{"t_s":24,"rss_mib":591.91796875},{"t_s":25,"rss_mib":591.91796875},{"t_s":25,"rss_mib":591.91796875},{"t_s":25,"rss_mib":591.91796875},{"t_s":26,"rss_mib":591.91796875},{"t_s":26,"rss_mib":591.91796875},{"t_s":27,"rss_mib":591.91796875},{"t_s":27,"rss_mib":591.91796875},{"t_s":28,"rss_mib":591.91796875},{"t_s":28,"rss_mib":591.91796875},{"t_s":29,"rss_mib":591.91796875},{"t_s":30,"rss_mib":591.91796875},{"t_s":30,"rss_mib":591.91796875},{"t_s":30,"rss_mib":591.91796875},{"t_s":31,"rss_mib":591.91796875},{"t_s":31,"rss_mib":591.91796875},{"t_s":32,"rss_mib":591.91796875},{"t_s":32,"rss_mib":591.91796875},{"t_s":33,"rss_mib":591.91796875},{"t_s":33,"rss_mib":591.91796875},{"t_s":34,"rss_mib":591.91796875},{"t_s":35,"rss_mib":591.91796875},{"t_s":35,"rss_mib":591.91796875},{"t_s":35,"rss_mib":591.91796875},{"t_s":36,"rss_mib":591.91796875},{"t_s":36,"rss_mib":591.91796875},{"t_s":37,"rss_mib":591.91796875},{"t_s":37,"rss_mib":591.91796875},{"t_s":38,"rss_mib":591.91796875},{"t_s":38,"rss_mib":591.91796875},{"t_s":39,"rss_mib":591.91796875},{"t_s":40,"rss_mib":591.91796875},{"t_s":40,"rss_mib":591.91796875},{"t_s":40,"rss_mib":591.91796875},{"t_s":41,"rss_mib":591.91796875},{"t_s":41,"rss_mib":591.91796875},{"t_s":42,"rss_mib":591.91796875},{"t_s":42,"rss_mib":591.91796875},{"t_s":43,"rss_mib":591.91796875},{"t_s":43,"rss_mib":591.91796875},{"t_s":44,"rss_mib":591.91796875},{"t_s":45,"rss_mib":591.91796875},{"t_s":45,"rss_mib":591.91796875},{"t_s":45,"rss_mib":591.91796875},{"t_s":46,"rss_mib":591.91796875},{"t_s":46,"rss_mib":591.91796875},{"t_s":47,"rss_mib":591.91796875},{"t_s":47,"rss_mib":591.91796875},{"t_s":48,"rss_mib":591.91796875},{"t_s":48,"rss_mib":591.91796875},{"t_s":49,"rss_mib":591.91796875},{"t_s":50,"rss_mib":591.91796875},{"t_s":50,"rss_mib":591.91796875},{"t_s":50,"rss_mib":591.91796875},{"t_s":51,"rss_mib":591.91796875},{"t_s":51,"rss_mib":591.91796875},{"t_s":52,"rss_mib":591.91796875},{"t_s":52,"rss_mib":591.91796875},{"t_s":53,"rss_mib":591.91796875},{"t_s":53,"rss_mib":591.91796875},{"t_s":54,"rss_mib":591.91796875},{"t_s":55,"rss_mib":591.91796875},{"t_s":55,"rss_mib":591.91796875},{"t_s":55,"rss_mib":591.91796875},{"t_s":56,"rss_mib":591.91796875},{"t_s":56,"rss_mib":591.91796875},{"t_s":57,"rss_mib":591.91796875},{"t_s":57,"rss_mib":591.91796875},{"t_s":58,"rss_mib":591.91796875},{"t_s":59,"rss_mib":591.91796875},{"t_s":59,"rss_mib":591.91796875},{"t_s":59,"rss_mib":591.91796875}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.008506517,"cost":6.119516938,"memory":440.764086394,"streaming":0.000005736,"streams_sustained":0.000001839,"throughput":481.687048359},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c2048.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}},"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10313,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":11125,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":11153,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10411.663807890223,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":96.04612850082371,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2500487063572403,"certified":true,"suppressed":false},"cost_window_ok":{"value":583,"certified":true,"suppressed":false},"cost_window_rps":{"value":97,"certified":true,"suppressed":false},"cost_threads":{"value":28,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.03773584905660377,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":97,"certified":true,"suppressed":false},"concurrency":1,"p99_us":10954,"first_disqualified_conc":8,"lower_bound":false,"samples":586},{"bound_ms":100,"rps":{"value":98,"certified":true,"suppressed":false},"concurrency":8,"p99_us":88352,"first_disqualified_conc":16,"lower_bound":false,"samples":589},{"bound_ms":null,"rps":{"value":98,"certified":true,"suppressed":false},"concurrency":8,"p99_us":88352,"first_disqualified_conc":16384,"lower_bound":false,"samples":589}],"sweep":[{"conc":1,"ok":588,"rps":97,"p99_us":10929,"fail":0},{"conc":1,"ok":588,"rps":97,"p99_us":10960,"fail":0},{"conc":1,"ok":586,"rps":97,"p99_us":10954,"fail":0},{"conc":2,"ok":582,"rps":96,"p99_us":21462,"fail":0},{"conc":2,"ok":585,"rps":97,"p99_us":21417,"fail":0},{"conc":2,"ok":588,"rps":97,"p99_us":21224,"fail":0},{"conc":4,"ok":582,"rps":96,"p99_us":42538,"fail":0},{"conc":4,"ok":586,"rps":97,"p99_us":44453,"fail":0},{"conc":4,"ok":588,"rps":97,"p99_us":43436,"fail":0},{"conc":8,"ok":587,"rps":97,"p99_us":88971,"fail":0},{"conc":8,"ok":592,"rps":98,"p99_us":88205,"fail":0},{"conc":8,"ok":589,"rps":98,"p99_us":88352,"fail":0},{"conc":16,"ok":585,"rps":97,"p99_us":183017,"fail":0},{"conc":16,"ok":584,"rps":97,"p99_us":178988,"fail":0},{"conc":16,"ok":584,"rps":97,"p99_us":180944,"fail":0},{"conc":32,"ok":584,"rps":97,"p99_us":500699,"fail":0},{"conc":32,"ok":582,"rps":96,"p99_us":502209,"fail":0},{"conc":32,"ok":580,"rps":96,"p99_us":511803,"fail":0},{"conc":64,"ok":579,"rps":96,"p99_us":1776639,"fail":0},{"conc":64,"ok":577,"rps":96,"p99_us":1773863,"fail":0},{"conc":64,"ok":532,"rps":88,"p99_us":1751944,"fail":0},{"conc":128,"ok":561,"rps":93,"p99_us":5475208,"fail":0},{"conc":128,"ok":565,"rps":94,"p99_us":5593079,"fail":0},{"conc":128,"ok":515,"rps":85,"p99_us":5600357,"fail":0},{"conc":256,"ok":469,"rps":78,"p99_us":5100753,"fail":0},{"conc":256,"ok":570,"rps":94,"p99_us":5490588,"fail":0},{"conc":256,"ok":516,"rps":85,"p99_us":5029102,"fail":0},{"conc":512,"ok":516,"rps":85,"p99_us":4424557,"fail":0},{"conc":512,"ok":407,"rps":67,"p99_us":3870196,"fail":0},{"conc":512,"ok":393,"rps":65,"p99_us":4892944,"fail":0},{"conc":1024,"ok":370,"rps":61,"p99_us":3844504,"fail":0},{"conc":1024,"ok":371,"rps":61,"p99_us":4104111,"fail":0},{"conc":1024,"ok":345,"rps":57,"p99_us":4855788,"fail":0},{"conc":2048,"ok":561,"rps":93,"p99_us":5461153,"fail":0},{"conc":2048,"ok":384,"rps":63,"p99_us":4595409,"fail":0},{"conc":2048,"ok":378,"rps":62,"p99_us":4541522,"fail":0},{"conc":4096,"ok":370,"rps":61,"p99_us":4440023,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":283},{"conc":8192,"ok":440,"rps":73,"p99_us":5514919,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":384},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":10172},{"conc":16384,"ok":114,"rps":18,"p99_us":5963133,"fail":746},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":663}],"rungs":[{"conc":1,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":10954,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":21417,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":43436,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":98,"certified":true,"suppressed":false},"p99_us":88352,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":180944,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":96,"certified":true,"suppressed":false},"p99_us":502209,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":96,"certified":true,"suppressed":false},"p99_us":1773863,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":93,"certified":true,"suppressed":false},"p99_us":5593079,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5100753,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":67,"certified":true,"suppressed":false},"p99_us":4424557,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":4104111,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":4595409,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":4440023,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":73,"certified":true,"suppressed":false},"p99_us":5514919,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 579 successful gateway round trip(s) and 265135 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":579,"direct_c1_samples":265135},"stream":{"stream_served":true,"stream_error":null,"reason":null,"stream_c1_note":"the c=1 streaming legs read 64 frame(s) through the gateway and 64 direct to the mock, out of a 64 frame budget. The added-GAP figures are percentiles over the intervals inside those two streams. The added-TTFT figures are percentiles over separate probes - 100 through the gateway and 100 direct, out of 100 attempted per leg - so they carry that weight and no more","added_ttft_p50_us":{"value":13428,"certified":true,"suppressed":false},"added_ttft_p99_us":{"value":14151,"certified":true,"suppressed":false},"added_gap_p50_us":{"value":3,"certified":true,"suppressed":false},"added_gap_p99_us":{"value":1348,"certified":true,"suppressed":false},"streams_sustained_fps":{"value":1232.8623220270053,"certified":true,"suppressed":false,"headroom":0.20745279457185184,"rig_ceiling":5942.857142857143},"streams_sustained":{"value":117,"certified":true,"suppressed":false,"headroom":0.20745279457185184,"rig_ceiling":5942.857142857143},"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.48828125,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":667.12109375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":304,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.48828125},{"t_s":2,"rss_mib":666.6875},{"t_s":2,"rss_mib":666.69140625},{"t_s":5,"rss_mib":666.921875},{"t_s":6,"rss_mib":666.921875},{"t_s":8,"rss_mib":667.01953125},{"t_s":9,"rss_mib":667.0234375},{"t_s":11,"rss_mib":667.078125},{"t_s":12,"rss_mib":667.078125},{"t_s":14,"rss_mib":667.1171875},{"t_s":15,"rss_mib":667.1171875},{"t_s":16,"rss_mib":667.12109375},{"t_s":18,"rss_mib":667.12109375},{"t_s":21,"rss_mib":667.12109375},{"t_s":24,"rss_mib":667.12109375},{"t_s":27,"rss_mib":667.12109375},{"t_s":30,"rss_mib":667.12109375},{"t_s":33,"rss_mib":667.12109375},{"t_s":36,"rss_mib":667.12109375},{"t_s":39,"rss_mib":667.12109375},{"t_s":42,"rss_mib":667.12109375},{"t_s":45,"rss_mib":667.12109375},{"t_s":48,"rss_mib":667.12109375},{"t_s":51,"rss_mib":667.12109375},{"t_s":54,"rss_mib":667.12109375},{"t_s":57,"rss_mib":667.12109375},{"t_s":60,"rss_mib":667.12109375},{"t_s":63,"rss_mib":667.12109375},{"t_s":66,"rss_mib":667.12109375},{"t_s":69,"rss_mib":667.12109375},{"t_s":72,"rss_mib":667.12109375},{"t_s":76,"rss_mib":667.12109375},{"t_s":79,"rss_mib":667.12109375},{"t_s":82,"rss_mib":667.12109375},{"t_s":85,"rss_mib":667.12109375},{"t_s":88,"rss_mib":667.12109375},{"t_s":91,"rss_mib":667.12109375},{"t_s":94,"rss_mib":667.12109375},{"t_s":97,"rss_mib":667.12109375},{"t_s":100,"rss_mib":667.12109375},{"t_s":103,"rss_mib":667.12109375},{"t_s":106,"rss_mib":667.12109375},{"t_s":109,"rss_mib":667.12109375},{"t_s":112,"rss_mib":667.12109375},{"t_s":115,"rss_mib":667.12109375},{"t_s":118,"rss_mib":667.12109375},{"t_s":121,"rss_mib":667.12109375},{"t_s":124,"rss_mib":667.12109375},{"t_s":127,"rss_mib":667.12109375},{"t_s":130,"rss_mib":667.12109375},{"t_s":133,"rss_mib":667.12109375},{"t_s":136,"rss_mib":667.12109375},{"t_s":139,"rss_mib":667.12109375},{"t_s":142,"rss_mib":667.12109375},{"t_s":145,"rss_mib":667.12109375},{"t_s":149,"rss_mib":667.12109375},{"t_s":152,"rss_mib":667.12109375},{"t_s":155,"rss_mib":667.12109375},{"t_s":158,"rss_mib":667.12109375},{"t_s":161,"rss_mib":667.12109375},{"t_s":164,"rss_mib":667.12109375},{"t_s":167,"rss_mib":667.12109375},{"t_s":170,"rss_mib":667.12109375},{"t_s":173,"rss_mib":667.12109375},{"t_s":176,"rss_mib":667.12109375},{"t_s":179,"rss_mib":667.12109375},{"t_s":182,"rss_mib":667.12109375},{"t_s":185,"rss_mib":667.12109375},{"t_s":188,"rss_mib":667.12109375},{"t_s":191,"rss_mib":667.12109375},{"t_s":194,"rss_mib":667.12109375},{"t_s":197,"rss_mib":667.12109375},{"t_s":200,"rss_mib":667.12109375},{"t_s":203,"rss_mib":667.12109375},{"t_s":206,"rss_mib":667.12109375},{"t_s":209,"rss_mib":667.12109375},{"t_s":212,"rss_mib":667.12109375},{"t_s":215,"rss_mib":667.12109375},{"t_s":219,"rss_mib":667.12109375},{"t_s":221,"rss_mib":667.12109375},{"t_s":225,"rss_mib":667.12109375},{"t_s":228,"rss_mib":667.12109375},{"t_s":231,"rss_mib":667.12109375},{"t_s":234,"rss_mib":667.12109375},{"t_s":237,"rss_mib":667.12109375},{"t_s":240,"rss_mib":667.12109375},{"t_s":243,"rss_mib":667.12109375},{"t_s":246,"rss_mib":667.12109375},{"t_s":249,"rss_mib":667.12109375},{"t_s":252,"rss_mib":667.12109375},{"t_s":255,"rss_mib":667.12109375},{"t_s":258,"rss_mib":667.12109375},{"t_s":261,"rss_mib":667.12109375},{"t_s":264,"rss_mib":667.12109375},{"t_s":267,"rss_mib":667.12109375},{"t_s":270,"rss_mib":667.12109375},{"t_s":273,"rss_mib":667.12109375},{"t_s":276,"rss_mib":667.12109375},{"t_s":279,"rss_mib":667.12109375},{"t_s":282,"rss_mib":667.12109375},{"t_s":285,"rss_mib":667.12109375},{"t_s":288,"rss_mib":667.12109375},{"t_s":291,"rss_mib":667.12109375},{"t_s":295,"rss_mib":667.12109375},{"t_s":298,"rss_mib":667.12109375},{"t_s":301,"rss_mib":667.12109375},{"t_s":304,"rss_mib":667.12109375},{"t_s":307,"rss_mib":667.12109375},{"t_s":310,"rss_mib":667.12109375},{"t_s":313,"rss_mib":667.12109375},{"t_s":316,"rss_mib":667.12109375},{"t_s":319,"rss_mib":667.12109375},{"t_s":322,"rss_mib":667.12109375},{"t_s":325,"rss_mib":667.12109375},{"t_s":328,"rss_mib":667.12109375},{"t_s":331,"rss_mib":667.12109375},{"t_s":334,"rss_mib":667.12109375},{"t_s":337,"rss_mib":667.12109375},{"t_s":340,"rss_mib":667.12109375},{"t_s":343,"rss_mib":667.12109375},{"t_s":346,"rss_mib":667.12109375},{"t_s":349,"rss_mib":667.12109375},{"t_s":352,"rss_mib":667.12109375},{"t_s":355,"rss_mib":667.12109375},{"t_s":358,"rss_mib":667.12109375},{"t_s":361,"rss_mib":667.12109375},{"t_s":364,"rss_mib":667.12109375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.48828125},{"t_s":0,"rss_mib":592.48828125},{"t_s":0,"rss_mib":592.48828125},{"t_s":1,"rss_mib":592.48828125},{"t_s":1,"rss_mib":592.48828125},{"t_s":2,"rss_mib":592.48828125},{"t_s":2,"rss_mib":592.48828125},{"t_s":3,"rss_mib":592.48828125},{"t_s":3,"rss_mib":592.48828125},{"t_s":4,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":6,"rss_mib":592.48828125},{"t_s":6,"rss_mib":592.48828125},{"t_s":7,"rss_mib":592.48828125},{"t_s":7,"rss_mib":592.48828125},{"t_s":8,"rss_mib":592.48828125},{"t_s":8,"rss_mib":592.48828125},{"t_s":9,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":11,"rss_mib":592.48828125},{"t_s":11,"rss_mib":592.48828125},{"t_s":12,"rss_mib":592.48828125},{"t_s":12,"rss_mib":592.48828125},{"t_s":13,"rss_mib":592.48828125},{"t_s":13,"rss_mib":592.48828125},{"t_s":14,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":16,"rss_mib":592.48828125},{"t_s":16,"rss_mib":592.48828125},{"t_s":17,"rss_mib":592.48828125},{"t_s":17,"rss_mib":592.48828125},{"t_s":18,"rss_mib":592.48828125},{"t_s":18,"rss_mib":592.48828125},{"t_s":19,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":21,"rss_mib":592.48828125},{"t_s":21,"rss_mib":592.48828125},{"t_s":22,"rss_mib":592.48828125},{"t_s":22,"rss_mib":592.48828125},{"t_s":23,"rss_mib":592.48828125},{"t_s":23,"rss_mib":592.48828125},{"t_s":24,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":26,"rss_mib":592.48828125},{"t_s":26,"rss_mib":592.48828125},{"t_s":27,"rss_mib":592.48828125},{"t_s":27,"rss_mib":592.48828125},{"t_s":28,"rss_mib":592.48828125},{"t_s":28,"rss_mib":592.48828125},{"t_s":29,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":31,"rss_mib":592.48828125},{"t_s":31,"rss_mib":592.48828125},{"t_s":32,"rss_mib":592.48828125},{"t_s":32,"rss_mib":592.48828125},{"t_s":33,"rss_mib":592.48828125},{"t_s":33,"rss_mib":592.48828125},{"t_s":34,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":36,"rss_mib":592.48828125},{"t_s":36,"rss_mib":592.48828125},{"t_s":37,"rss_mib":592.48828125},{"t_s":37,"rss_mib":592.48828125},{"t_s":38,"rss_mib":592.48828125},{"t_s":38,"rss_mib":592.48828125},{"t_s":39,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":41,"rss_mib":592.48828125},{"t_s":41,"rss_mib":592.48828125},{"t_s":42,"rss_mib":592.48828125},{"t_s":42,"rss_mib":592.48828125},{"t_s":43,"rss_mib":592.48828125},{"t_s":43,"rss_mib":592.48828125},{"t_s":44,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":46,"rss_mib":592.48828125},{"t_s":46,"rss_mib":592.48828125},{"t_s":47,"rss_mib":592.48828125},{"t_s":47,"rss_mib":592.48828125},{"t_s":48,"rss_mib":592.48828125},{"t_s":48,"rss_mib":592.48828125},{"t_s":49,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":51,"rss_mib":592.48828125},{"t_s":51,"rss_mib":592.48828125},{"t_s":52,"rss_mib":592.48828125},{"t_s":52,"rss_mib":592.48828125},{"t_s":53,"rss_mib":592.48828125},{"t_s":53,"rss_mib":592.48828125},{"t_s":54,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":56,"rss_mib":592.48828125},{"t_s":56,"rss_mib":592.48828125},{"t_s":57,"rss_mib":592.48828125},{"t_s":57,"rss_mib":592.48828125},{"t_s":58,"rss_mib":592.48828125},{"t_s":58,"rss_mib":592.48828125},{"t_s":59,"rss_mib":592.48828125},{"t_s":59,"rss_mib":592.48828125}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.010464049,"cost":6.082499035,"memory":440.381291728,"streaming":4.020506158,"streams_sustained":66.994876574,"throughput":639.211182589},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"}}},"anthropic":{"served":true,"reason":null,"status":"","path":"/v1/messages","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10994,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":11745,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":11773,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":11322.160148975792,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":88.32236842105263,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2501294799366864,"certified":true,"suppressed":false},"cost_window_ok":{"value":537,"certified":true,"suppressed":false},"cost_window_rps":{"value":89,"certified":true,"suppressed":false},"cost_threads":{"value":268,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.03910614525139665,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":89,"certified":true,"suppressed":false},"concurrency":2,"p99_us":23288,"first_disqualified_conc":8,"lower_bound":false,"samples":538},{"bound_ms":100,"rps":{"value":89,"certified":true,"suppressed":false},"concurrency":2,"p99_us":23288,"first_disqualified_conc":16,"lower_bound":false,"samples":538},{"bound_ms":null,"rps":{"value":91,"certified":true,"suppressed":false},"concurrency":64,"p99_us":796228,"first_disqualified_conc":4096,"lower_bound":false,"samples":550}],"sweep":[{"conc":1,"ok":534,"rps":88,"p99_us":11990,"fail":0},{"conc":1,"ok":534,"rps":88,"p99_us":12020,"fail":0},{"conc":1,"ok":534,"rps":88,"p99_us":12004,"fail":0},{"conc":2,"ok":540,"rps":89,"p99_us":23294,"fail":0},{"conc":2,"ok":540,"rps":89,"p99_us":23082,"fail":0},{"conc":2,"ok":538,"rps":89,"p99_us":23288,"fail":0},{"conc":4,"ok":535,"rps":89,"p99_us":47845,"fail":0},{"conc":4,"ok":536,"rps":89,"p99_us":46755,"fail":0},{"conc":4,"ok":534,"rps":88,"p99_us":46942,"fail":0},{"conc":8,"ok":538,"rps":89,"p99_us":96490,"fail":0},{"conc":8,"ok":538,"rps":89,"p99_us":96016,"fail":0},{"conc":8,"ok":538,"rps":89,"p99_us":96548,"fail":0},{"conc":16,"ok":540,"rps":89,"p99_us":195324,"fail":0},{"conc":16,"ok":539,"rps":89,"p99_us":194181,"fail":0},{"conc":16,"ok":540,"rps":89,"p99_us":196938,"fail":0},{"conc":32,"ok":541,"rps":90,"p99_us":522479,"fail":0},{"conc":32,"ok":540,"rps":89,"p99_us":536677,"fail":0},{"conc":32,"ok":540,"rps":89,"p99_us":538075,"fail":0},{"conc":64,"ok":550,"rps":91,"p99_us":796228,"fail":0},{"conc":64,"ok":496,"rps":82,"p99_us":2423384,"fail":0},{"conc":64,"ok":541,"rps":90,"p99_us":1877686,"fail":0},{"conc":128,"ok":524,"rps":87,"p99_us":5509859,"fail":0},{"conc":128,"ok":512,"rps":85,"p99_us":1895571,"fail":0},{"conc":128,"ok":524,"rps":87,"p99_us":5392638,"fail":0},{"conc":256,"ok":519,"rps":86,"p99_us":5420133,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":2906765,"fail":0},{"conc":256,"ok":508,"rps":84,"p99_us":5444252,"fail":0},{"conc":512,"ok":528,"rps":87,"p99_us":5371375,"fail":0},{"conc":512,"ok":490,"rps":81,"p99_us":4812547,"fail":0},{"conc":512,"ok":383,"rps":63,"p99_us":4973182,"fail":0},{"conc":1024,"ok":480,"rps":79,"p99_us":5166944,"fail":0},{"conc":1024,"ok":369,"rps":61,"p99_us":4393460,"fail":0},{"conc":1024,"ok":362,"rps":60,"p99_us":4959591,"fail":0},{"conc":2048,"ok":523,"rps":87,"p99_us":5353812,"fail":0},{"conc":2048,"ok":371,"rps":61,"p99_us":5139912,"fail":0},{"conc":2048,"ok":377,"rps":61,"p99_us":4544692,"fail":0},{"conc":4096,"ok":373,"rps":61,"p99_us":4609082,"fail":343},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":88,"certified":true,"suppressed":false},"p99_us":12004,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":23288,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":46942,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":96490,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":195324,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":89,"certified":true,"suppressed":false},"p99_us":536677,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":90,"certified":true,"suppressed":false},"p99_us":1877686,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":5392638,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5420133,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":81,"certified":true,"suppressed":false},"p99_us":4973182,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":4959591,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":5139912,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 1 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"anthropic>anthropic is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 544 successful gateway round trip(s) and 263642 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":544,"direct_c1_samples":263642},"stream":{"stream_served":true,"stream_error":null,"reason":null,"stream_c1_note":"the c=1 streaming legs read 64 frame(s) through the gateway and 64 direct to the mock, out of a 64 frame budget. The added-GAP figures are percentiles over the intervals inside those two streams. The added-TTFT figures are percentiles over separate probes - 100 through the gateway and 100 direct, out of 100 attempted per leg - so they carry that weight and no more","added_ttft_p50_us":{"value":14510,"certified":true,"suppressed":false},"added_ttft_p99_us":{"value":15815,"certified":true,"suppressed":false},"added_gap_p50_us":{"value":0,"certified":true,"suppressed":false},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21123us) came in under the mock's (21136us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing"},"streams_sustained_fps":{"value":3187.580899184528,"certified":true,"suppressed":false,"headroom":0.2460999958929231,"rig_ceiling":12952.380952380952},"streams_sustained":{"value":255,"certified":true,"suppressed":false,"headroom":0.2460999958929231,"rig_ceiling":12952.380952380952},"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.48828125,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":658.05859375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":300,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.48828125},{"t_s":2,"rss_mib":649.99609375},{"t_s":2,"rss_mib":649.99609375},{"t_s":5,"rss_mib":649.99609375},{"t_s":8,"rss_mib":649.99609375},{"t_s":11,"rss_mib":649.99609375},{"t_s":15,"rss_mib":649.99609375},{"t_s":15,"rss_mib":652.01953125},{"t_s":17,"rss_mib":652.01953125},{"t_s":20,"rss_mib":652.01953125},{"t_s":24,"rss_mib":652.01953125},{"t_s":26,"rss_mib":652.01953125},{"t_s":30,"rss_mib":652.01953125},{"t_s":33,"rss_mib":652.01953125},{"t_s":36,"rss_mib":652.01953125},{"t_s":36,"rss_mib":652.0703125},{"t_s":39,"rss_mib":652.0703125},{"t_s":39,"rss_mib":654.09375},{"t_s":42,"rss_mib":654.09375},{"t_s":45,"rss_mib":654.09375},{"t_s":48,"rss_mib":654.09375},{"t_s":49,"rss_mib":654.23828125},{"t_s":51,"rss_mib":654.23828125},{"t_s":53,"rss_mib":654.2578125},{"t_s":54,"rss_mib":654.2578125},{"t_s":56,"rss_mib":654.3515625},{"t_s":57,"rss_mib":654.3515625},{"t_s":58,"rss_mib":654.3828125},{"t_s":60,"rss_mib":654.3828125},{"t_s":62,"rss_mib":654.56640625},{"t_s":63,"rss_mib":654.56640625},{"t_s":64,"rss_mib":654.578125},{"t_s":66,"rss_mib":654.578125},{"t_s":68,"rss_mib":654.68359375},{"t_s":69,"rss_mib":654.68359375},{"t_s":71,"rss_mib":654.69921875},{"t_s":72,"rss_mib":654.69921875},{"t_s":74,"rss_mib":654.80078125},{"t_s":75,"rss_mib":654.80078125},{"t_s":77,"rss_mib":654.80859375},{"t_s":78,"rss_mib":654.81640625},{"t_s":80,"rss_mib":654.82421875},{"t_s":81,"rss_mib":654.82421875},{"t_s":83,"rss_mib":654.828125},{"t_s":84,"rss_mib":654.828125},{"t_s":86,"rss_mib":654.9609375},{"t_s":87,"rss_mib":654.96484375},{"t_s":90,"rss_mib":654.96484375},{"t_s":93,"rss_mib":654.96484375},{"t_s":96,"rss_mib":654.96484375},{"t_s":99,"rss_mib":654.96484375},{"t_s":102,"rss_mib":654.96484375},{"t_s":105,"rss_mib":654.96484375},{"t_s":106,"rss_mib":656.98828125},{"t_s":108,"rss_mib":656.98828125},{"t_s":111,"rss_mib":656.98828125},{"t_s":114,"rss_mib":656.98828125},{"t_s":117,"rss_mib":656.98828125},{"t_s":120,"rss_mib":656.98828125},{"t_s":123,"rss_mib":656.98828125},{"t_s":126,"rss_mib":656.98828125},{"t_s":129,"rss_mib":656.98828125},{"t_s":132,"rss_mib":656.98828125},{"t_s":135,"rss_mib":656.98828125},{"t_s":138,"rss_mib":656.98828125},{"t_s":141,"rss_mib":656.98828125},{"t_s":144,"rss_mib":656.98828125},{"t_s":147,"rss_mib":656.98828125},{"t_s":150,"rss_mib":656.98828125},{"t_s":153,"rss_mib":656.98828125},{"t_s":155,"rss_mib":657},{"t_s":156,"rss_mib":657},{"t_s":159,"rss_mib":657},{"t_s":162,"rss_mib":657},{"t_s":164,"rss_mib":657.00390625},{"t_s":165,"rss_mib":657.00390625},{"t_s":168,"rss_mib":657.00390625},{"t_s":171,"rss_mib":657.00390625},{"t_s":174,"rss_mib":657.015625},{"t_s":174,"rss_mib":657.015625},{"t_s":175,"rss_mib":657.1875},{"t_s":177,"rss_mib":657.1875},{"t_s":177,"rss_mib":657.19140625},{"t_s":180,"rss_mib":657.19140625},{"t_s":180,"rss_mib":657.1953125},{"t_s":183,"rss_mib":657.1953125},{"t_s":183,"rss_mib":657.203125},{"t_s":186,"rss_mib":657.203125},{"t_s":187,"rss_mib":657.39453125},{"t_s":189,"rss_mib":657.39453125},{"t_s":189,"rss_mib":657.3984375},{"t_s":192,"rss_mib":657.3984375},{"t_s":193,"rss_mib":657.4375},{"t_s":195,"rss_mib":657.4375},{"t_s":197,"rss_mib":657.453125},{"t_s":198,"rss_mib":657.453125},{"t_s":200,"rss_mib":657.5},{"t_s":201,"rss_mib":657.5},{"t_s":203,"rss_mib":657.53125},{"t_s":204,"rss_mib":657.53125},{"t_s":207,"rss_mib":657.53125},{"t_s":210,"rss_mib":657.71875},{"t_s":210,"rss_mib":657.71875},{"t_s":210,"rss_mib":657.73046875},{"t_s":213,"rss_mib":657.73046875},{"t_s":216,"rss_mib":657.78125},{"t_s":216,"rss_mib":657.81640625},{"t_s":218,"rss_mib":657.9453125},{"t_s":219,"rss_mib":657.9453125},{"t_s":222,"rss_mib":657.96484375},{"t_s":222,"rss_mib":657.984375},{"t_s":222,"rss_mib":658.05859375},{"t_s":225,"rss_mib":658.05859375},{"t_s":228,"rss_mib":658.05859375},{"t_s":231,"rss_mib":658.05859375},{"t_s":234,"rss_mib":658.05859375},{"t_s":237,"rss_mib":658.05859375},{"t_s":240,"rss_mib":658.05859375},{"t_s":243,"rss_mib":658.05859375},{"t_s":246,"rss_mib":658.05859375},{"t_s":249,"rss_mib":658.05859375},{"t_s":252,"rss_mib":658.05859375},{"t_s":255,"rss_mib":658.05859375},{"t_s":258,"rss_mib":658.05859375},{"t_s":261,"rss_mib":658.05859375},{"t_s":264,"rss_mib":658.05859375},{"t_s":267,"rss_mib":658.05859375},{"t_s":270,"rss_mib":658.05859375},{"t_s":273,"rss_mib":658.05859375},{"t_s":276,"rss_mib":658.05859375},{"t_s":279,"rss_mib":658.05859375},{"t_s":282,"rss_mib":658.05859375},{"t_s":285,"rss_mib":658.05859375},{"t_s":288,"rss_mib":658.05859375},{"t_s":291,"rss_mib":658.05859375},{"t_s":294,"rss_mib":658.05859375},{"t_s":297,"rss_mib":658.05859375},{"t_s":300,"rss_mib":658.05859375},{"t_s":303,"rss_mib":658.05859375},{"t_s":306,"rss_mib":658.05859375},{"t_s":309,"rss_mib":658.05859375},{"t_s":312,"rss_mib":658.05859375},{"t_s":315,"rss_mib":658.05859375},{"t_s":318,"rss_mib":658.05859375},{"t_s":321,"rss_mib":658.05859375},{"t_s":324,"rss_mib":658.05859375},{"t_s":327,"rss_mib":658.05859375},{"t_s":330,"rss_mib":658.05859375},{"t_s":333,"rss_mib":658.05859375},{"t_s":336,"rss_mib":658.05859375},{"t_s":339,"rss_mib":658.05859375},{"t_s":342,"rss_mib":658.05859375},{"t_s":345,"rss_mib":658.05859375},{"t_s":348,"rss_mib":658.05859375},{"t_s":351,"rss_mib":658.05859375},{"t_s":354,"rss_mib":658.05859375},{"t_s":357,"rss_mib":658.05859375},{"t_s":360,"rss_mib":658.05859375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.48828125},{"t_s":0,"rss_mib":592.48828125},{"t_s":0,"rss_mib":592.48828125},{"t_s":1,"rss_mib":592.48828125},{"t_s":1,"rss_mib":592.48828125},{"t_s":2,"rss_mib":592.48828125},{"t_s":2,"rss_mib":592.48828125},{"t_s":3,"rss_mib":592.48828125},{"t_s":3,"rss_mib":592.48828125},{"t_s":4,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":5,"rss_mib":592.48828125},{"t_s":6,"rss_mib":592.48828125},{"t_s":6,"rss_mib":592.48828125},{"t_s":7,"rss_mib":592.48828125},{"t_s":7,"rss_mib":592.48828125},{"t_s":8,"rss_mib":592.48828125},{"t_s":8,"rss_mib":592.48828125},{"t_s":9,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":10,"rss_mib":592.48828125},{"t_s":11,"rss_mib":592.48828125},{"t_s":11,"rss_mib":592.48828125},{"t_s":12,"rss_mib":592.48828125},{"t_s":12,"rss_mib":592.48828125},{"t_s":13,"rss_mib":592.48828125},{"t_s":13,"rss_mib":592.48828125},{"t_s":14,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":15,"rss_mib":592.48828125},{"t_s":16,"rss_mib":592.48828125},{"t_s":16,"rss_mib":592.48828125},{"t_s":17,"rss_mib":592.48828125},{"t_s":17,"rss_mib":592.48828125},{"t_s":18,"rss_mib":592.48828125},{"t_s":18,"rss_mib":592.48828125},{"t_s":19,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":20,"rss_mib":592.48828125},{"t_s":21,"rss_mib":592.48828125},{"t_s":21,"rss_mib":592.48828125},{"t_s":22,"rss_mib":592.48828125},{"t_s":22,"rss_mib":592.48828125},{"t_s":23,"rss_mib":592.48828125},{"t_s":23,"rss_mib":592.48828125},{"t_s":24,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":25,"rss_mib":592.48828125},{"t_s":26,"rss_mib":592.48828125},{"t_s":26,"rss_mib":592.48828125},{"t_s":27,"rss_mib":592.48828125},{"t_s":27,"rss_mib":592.48828125},{"t_s":28,"rss_mib":592.48828125},{"t_s":28,"rss_mib":592.48828125},{"t_s":29,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":30,"rss_mib":592.48828125},{"t_s":31,"rss_mib":592.48828125},{"t_s":31,"rss_mib":592.48828125},{"t_s":32,"rss_mib":592.48828125},{"t_s":32,"rss_mib":592.48828125},{"t_s":33,"rss_mib":592.48828125},{"t_s":33,"rss_mib":592.48828125},{"t_s":34,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":35,"rss_mib":592.48828125},{"t_s":36,"rss_mib":592.48828125},{"t_s":36,"rss_mib":592.48828125},{"t_s":37,"rss_mib":592.48828125},{"t_s":37,"rss_mib":592.48828125},{"t_s":38,"rss_mib":592.48828125},{"t_s":38,"rss_mib":592.48828125},{"t_s":39,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":40,"rss_mib":592.48828125},{"t_s":41,"rss_mib":592.48828125},{"t_s":41,"rss_mib":592.48828125},{"t_s":42,"rss_mib":592.48828125},{"t_s":42,"rss_mib":592.48828125},{"t_s":43,"rss_mib":592.48828125},{"t_s":43,"rss_mib":592.48828125},{"t_s":44,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":45,"rss_mib":592.48828125},{"t_s":46,"rss_mib":592.48828125},{"t_s":46,"rss_mib":592.48828125},{"t_s":47,"rss_mib":592.48828125},{"t_s":47,"rss_mib":592.48828125},{"t_s":48,"rss_mib":592.48828125},{"t_s":49,"rss_mib":592.48828125},{"t_s":49,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":50,"rss_mib":592.48828125},{"t_s":51,"rss_mib":592.48828125},{"t_s":51,"rss_mib":592.48828125},{"t_s":52,"rss_mib":592.48828125},{"t_s":52,"rss_mib":592.48828125},{"t_s":53,"rss_mib":592.48828125},{"t_s":54,"rss_mib":592.48828125},{"t_s":54,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":55,"rss_mib":592.48828125},{"t_s":56,"rss_mib":592.48828125},{"t_s":56,"rss_mib":592.48828125},{"t_s":57,"rss_mib":592.48828125},{"t_s":57,"rss_mib":592.48828125},{"t_s":58,"rss_mib":592.48828125},{"t_s":59,"rss_mib":592.48828125},{"t_s":59,"rss_mib":592.48828125},{"t_s":59,"rss_mib":592.48828125}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.018873445,"cost":6.089756816,"memory":436.1056508,"streaming":4.060487701,"streams_sustained":63.3246658,"throughput":442.337776599},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p99_us":{"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21123us) came in under the mock's (21136us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing"}}}}},"bedrock":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":17646,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":18361,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":18388,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":18244.04761904762,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":54.81239804241436,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2502884887591485,"certified":true,"suppressed":false},"cost_window_ok":{"value":336,"certified":true,"suppressed":false},"cost_window_rps":{"value":55,"certified":true,"suppressed":false},"cost_threads":{"value":17,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.0744047619047619,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":56,"certified":true,"suppressed":false},"concurrency":1,"p99_us":18407,"first_disqualified_conc":4,"lower_bound":false,"samples":338},{"bound_ms":100,"rps":{"value":56,"certified":true,"suppressed":false},"concurrency":1,"p99_us":18407,"first_disqualified_conc":8,"lower_bound":false,"samples":338},{"bound_ms":null,"rps":{"value":57,"certified":true,"suppressed":false},"concurrency":2048,"p99_us":5344748,"first_disqualified_conc":4096,"lower_bound":false,"samples":345}],"sweep":[{"conc":1,"ok":337,"rps":56,"p99_us":18530,"fail":0},{"conc":1,"ok":337,"rps":56,"p99_us":18394,"fail":0},{"conc":1,"ok":338,"rps":56,"p99_us":18407,"fail":0},{"conc":2,"ok":339,"rps":56,"p99_us":40168,"fail":0},{"conc":2,"ok":339,"rps":56,"p99_us":36620,"fail":0},{"conc":2,"ok":338,"rps":56,"p99_us":36463,"fail":0},{"conc":4,"ok":336,"rps":55,"p99_us":75509,"fail":0},{"conc":4,"ok":339,"rps":56,"p99_us":77350,"fail":0},{"conc":4,"ok":337,"rps":56,"p99_us":74262,"fail":0},{"conc":8,"ok":312,"rps":51,"p99_us":574237,"fail":0},{"conc":8,"ok":338,"rps":56,"p99_us":158768,"fail":0},{"conc":8,"ok":336,"rps":55,"p99_us":153276,"fail":0},{"conc":16,"ok":333,"rps":55,"p99_us":304309,"fail":0},{"conc":16,"ok":333,"rps":55,"p99_us":322822,"fail":0},{"conc":16,"ok":335,"rps":55,"p99_us":304562,"fail":0},{"conc":32,"ok":333,"rps":55,"p99_us":825509,"fail":0},{"conc":32,"ok":333,"rps":55,"p99_us":851845,"fail":0},{"conc":32,"ok":332,"rps":55,"p99_us":866046,"fail":0},{"conc":64,"ok":321,"rps":53,"p99_us":1252661,"fail":0},{"conc":64,"ok":322,"rps":53,"p99_us":2632313,"fail":0},{"conc":64,"ok":322,"rps":53,"p99_us":2631729,"fail":0},{"conc":128,"ok":312,"rps":51,"p99_us":5573570,"fail":0},{"conc":128,"ok":286,"rps":47,"p99_us":5711238,"fail":0},{"conc":128,"ok":309,"rps":51,"p99_us":2393685,"fail":0},{"conc":256,"ok":311,"rps":51,"p99_us":5634162,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":4804394,"fail":0},{"conc":256,"ok":293,"rps":48,"p99_us":5659177,"fail":0},{"conc":512,"ok":290,"rps":48,"p99_us":5741025,"fail":0},{"conc":512,"ok":312,"rps":51,"p99_us":5608468,"fail":0},{"conc":512,"ok":315,"rps":52,"p99_us":5651891,"fail":0},{"conc":1024,"ok":320,"rps":52,"p99_us":5553855,"fail":0},{"conc":1024,"ok":184,"rps":29,"p99_us":5856508,"fail":0},{"conc":1024,"ok":222,"rps":36,"p99_us":5515972,"fail":0},{"conc":2048,"ok":345,"rps":57,"p99_us":5344748,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":18407,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":36620,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":75509,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":158768,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":304562,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":851845,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":53,"certified":true,"suppressed":false},"p99_us":2631729,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":51,"certified":true,"suppressed":false},"p99_us":5573570,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":48,"certified":true,"suppressed":false},"p99_us":5634162,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":51,"certified":true,"suppressed":false},"p99_us":5651891,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":36,"certified":true,"suppressed":false},"p99_us":5553855,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":57,"certified":true,"suppressed":false},"p99_us":5344748,"windows":3,"clean_windows":1},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 0 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 339 successful gateway round trip(s) and 267586 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":339,"direct_c1_samples":267586},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.97265625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":689.4375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":689.4375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":689.4375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":689.4375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":64,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":301,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.97265625},{"t_s":2,"rss_mib":688.34375},{"t_s":2,"rss_mib":688.36328125},{"t_s":5,"rss_mib":688.66796875},{"t_s":5,"rss_mib":688.671875},{"t_s":8,"rss_mib":688.9140625},{"t_s":8,"rss_mib":688.9140625},{"t_s":11,"rss_mib":689.08203125},{"t_s":11,"rss_mib":689.08984375},{"t_s":14,"rss_mib":689.203125},{"t_s":15,"rss_mib":689.20703125},{"t_s":17,"rss_mib":689.265625},{"t_s":17,"rss_mib":689.265625},{"t_s":20,"rss_mib":689.30078125},{"t_s":21,"rss_mib":689.30078125},{"t_s":23,"rss_mib":689.32421875},{"t_s":24,"rss_mib":689.32421875},{"t_s":25,"rss_mib":689.34375},{"t_s":27,"rss_mib":689.34375},{"t_s":28,"rss_mib":689.35546875},{"t_s":30,"rss_mib":689.35546875},{"t_s":31,"rss_mib":689.3671875},{"t_s":33,"rss_mib":689.3671875},{"t_s":35,"rss_mib":689.37890625},{"t_s":36,"rss_mib":689.37890625},{"t_s":39,"rss_mib":689.37890625},{"t_s":41,"rss_mib":689.38671875},{"t_s":42,"rss_mib":689.38671875},{"t_s":45,"rss_mib":689.38671875},{"t_s":47,"rss_mib":689.39453125},{"t_s":48,"rss_mib":689.39453125},{"t_s":49,"rss_mib":689.4140625},{"t_s":51,"rss_mib":689.4140625},{"t_s":53,"rss_mib":689.421875},{"t_s":54,"rss_mib":689.421875},{"t_s":57,"rss_mib":689.421875},{"t_s":60,"rss_mib":689.421875},{"t_s":63,"rss_mib":689.421875},{"t_s":66,"rss_mib":689.421875},{"t_s":68,"rss_mib":689.4296875},{"t_s":69,"rss_mib":689.4296875},{"t_s":72,"rss_mib":689.4296875},{"t_s":73,"rss_mib":689.4375},{"t_s":75,"rss_mib":689.4375},{"t_s":78,"rss_mib":689.4375},{"t_s":81,"rss_mib":689.4375},{"t_s":84,"rss_mib":689.4375},{"t_s":87,"rss_mib":689.4375},{"t_s":90,"rss_mib":689.4375},{"t_s":93,"rss_mib":689.4375},{"t_s":96,"rss_mib":689.4375},{"t_s":99,"rss_mib":689.4375},{"t_s":102,"rss_mib":689.4375},{"t_s":105,"rss_mib":689.4375},{"t_s":108,"rss_mib":689.4375},{"t_s":111,"rss_mib":689.4375},{"t_s":114,"rss_mib":689.4375},{"t_s":117,"rss_mib":689.4375},{"t_s":120,"rss_mib":689.4375},{"t_s":123,"rss_mib":689.4375},{"t_s":126,"rss_mib":689.4375},{"t_s":129,"rss_mib":689.4375},{"t_s":132,"rss_mib":689.4375},{"t_s":135,"rss_mib":689.4375},{"t_s":138,"rss_mib":689.4375},{"t_s":141,"rss_mib":689.4375},{"t_s":144,"rss_mib":689.4375},{"t_s":147,"rss_mib":689.4375},{"t_s":150,"rss_mib":689.4375},{"t_s":153,"rss_mib":689.4375},{"t_s":156,"rss_mib":689.4375},{"t_s":159,"rss_mib":689.4375},{"t_s":162,"rss_mib":689.4375},{"t_s":165,"rss_mib":689.4375},{"t_s":168,"rss_mib":689.4375},{"t_s":171,"rss_mib":689.4375},{"t_s":174,"rss_mib":689.4375},{"t_s":177,"rss_mib":689.4375},{"t_s":180,"rss_mib":689.4375},{"t_s":183,"rss_mib":689.4375},{"t_s":186,"rss_mib":689.4375},{"t_s":189,"rss_mib":689.4375},{"t_s":192,"rss_mib":689.4375},{"t_s":195,"rss_mib":689.4375},{"t_s":198,"rss_mib":689.4375},{"t_s":201,"rss_mib":689.4375},{"t_s":204,"rss_mib":689.4375},{"t_s":207,"rss_mib":689.4375},{"t_s":210,"rss_mib":689.4375},{"t_s":213,"rss_mib":689.4375},{"t_s":216,"rss_mib":689.4375},{"t_s":219,"rss_mib":689.4375},{"t_s":222,"rss_mib":689.4375},{"t_s":225,"rss_mib":689.4375},{"t_s":228,"rss_mib":689.4375},{"t_s":231,"rss_mib":689.4375},{"t_s":234,"rss_mib":689.4375},{"t_s":237,"rss_mib":689.4375},{"t_s":240,"rss_mib":689.4375},{"t_s":243,"rss_mib":689.4375},{"t_s":246,"rss_mib":689.4375},{"t_s":249,"rss_mib":689.4375},{"t_s":252,"rss_mib":689.4375},{"t_s":255,"rss_mib":689.4375},{"t_s":258,"rss_mib":689.4375},{"t_s":261,"rss_mib":689.4375},{"t_s":264,"rss_mib":689.4375},{"t_s":267,"rss_mib":689.4375},{"t_s":270,"rss_mib":689.4375},{"t_s":273,"rss_mib":689.4375},{"t_s":276,"rss_mib":689.4375},{"t_s":279,"rss_mib":689.4375},{"t_s":282,"rss_mib":689.4375},{"t_s":285,"rss_mib":689.4375},{"t_s":288,"rss_mib":689.4375},{"t_s":292,"rss_mib":689.4375},{"t_s":294,"rss_mib":689.4375},{"t_s":298,"rss_mib":689.4375},{"t_s":301,"rss_mib":689.4375},{"t_s":304,"rss_mib":689.4375},{"t_s":307,"rss_mib":689.4375},{"t_s":310,"rss_mib":689.4375},{"t_s":313,"rss_mib":689.4375},{"t_s":316,"rss_mib":689.4375},{"t_s":319,"rss_mib":689.4375},{"t_s":322,"rss_mib":689.4375},{"t_s":325,"rss_mib":689.4375},{"t_s":328,"rss_mib":689.4375},{"t_s":331,"rss_mib":689.4375},{"t_s":334,"rss_mib":689.4375},{"t_s":337,"rss_mib":689.4375},{"t_s":340,"rss_mib":689.4375},{"t_s":343,"rss_mib":689.4375},{"t_s":346,"rss_mib":689.4375},{"t_s":349,"rss_mib":689.4375},{"t_s":352,"rss_mib":689.4375},{"t_s":355,"rss_mib":689.4375},{"t_s":358,"rss_mib":689.4375},{"t_s":361,"rss_mib":689.4375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.97265625},{"t_s":0,"rss_mib":592.97265625},{"t_s":0,"rss_mib":592.97265625},{"t_s":1,"rss_mib":592.97265625},{"t_s":1,"rss_mib":592.97265625},{"t_s":2,"rss_mib":592.97265625},{"t_s":2,"rss_mib":592.97265625},{"t_s":3,"rss_mib":592.97265625},{"t_s":3,"rss_mib":592.97265625},{"t_s":4,"rss_mib":592.97265625},{"t_s":5,"rss_mib":592.97265625},{"t_s":5,"rss_mib":592.97265625},{"t_s":5,"rss_mib":592.97265625},{"t_s":6,"rss_mib":592.97265625},{"t_s":6,"rss_mib":592.97265625},{"t_s":7,"rss_mib":592.97265625},{"t_s":7,"rss_mib":592.97265625},{"t_s":8,"rss_mib":592.97265625},{"t_s":8,"rss_mib":592.97265625},{"t_s":9,"rss_mib":592.97265625},{"t_s":10,"rss_mib":592.97265625},{"t_s":10,"rss_mib":592.97265625},{"t_s":10,"rss_mib":592.97265625},{"t_s":11,"rss_mib":592.97265625},{"t_s":11,"rss_mib":592.97265625},{"t_s":12,"rss_mib":592.97265625},{"t_s":12,"rss_mib":592.97265625},{"t_s":13,"rss_mib":592.97265625},{"t_s":13,"rss_mib":592.97265625},{"t_s":14,"rss_mib":592.97265625},{"t_s":15,"rss_mib":592.97265625},{"t_s":15,"rss_mib":592.97265625},{"t_s":16,"rss_mib":592.97265625},{"t_s":16,"rss_mib":592.97265625},{"t_s":16,"rss_mib":592.97265625},{"t_s":17,"rss_mib":592.97265625},{"t_s":17,"rss_mib":592.97265625},{"t_s":18,"rss_mib":592.97265625},{"t_s":18,"rss_mib":592.97265625},{"t_s":19,"rss_mib":592.97265625},{"t_s":20,"rss_mib":592.97265625},{"t_s":20,"rss_mib":592.97265625},{"t_s":21,"rss_mib":592.97265625},{"t_s":21,"rss_mib":592.97265625},{"t_s":21,"rss_mib":592.97265625},{"t_s":22,"rss_mib":592.97265625},{"t_s":22,"rss_mib":592.97265625},{"t_s":23,"rss_mib":592.97265625},{"t_s":23,"rss_mib":592.97265625},{"t_s":24,"rss_mib":592.97265625},{"t_s":25,"rss_mib":592.97265625},{"t_s":25,"rss_mib":592.97265625},{"t_s":26,"rss_mib":592.97265625},{"t_s":26,"rss_mib":592.97265625},{"t_s":27,"rss_mib":592.97265625},{"t_s":27,"rss_mib":592.97265625},{"t_s":27,"rss_mib":592.97265625},{"t_s":28,"rss_mib":592.97265625},{"t_s":28,"rss_mib":592.97265625},{"t_s":29,"rss_mib":592.97265625},{"t_s":30,"rss_mib":592.97265625},{"t_s":30,"rss_mib":592.97265625},{"t_s":31,"rss_mib":592.97265625},{"t_s":31,"rss_mib":592.97265625},{"t_s":32,"rss_mib":592.97265625},{"t_s":32,"rss_mib":592.97265625},{"t_s":32,"rss_mib":592.97265625},{"t_s":33,"rss_mib":592.97265625},{"t_s":33,"rss_mib":592.97265625},{"t_s":34,"rss_mib":592.97265625},{"t_s":35,"rss_mib":592.97265625},{"t_s":35,"rss_mib":592.97265625},{"t_s":36,"rss_mib":592.97265625},{"t_s":36,"rss_mib":592.97265625},{"t_s":37,"rss_mib":592.97265625},{"t_s":37,"rss_mib":592.97265625},{"t_s":38,"rss_mib":592.97265625},{"t_s":38,"rss_mib":592.97265625},{"t_s":38,"rss_mib":592.97265625},{"t_s":39,"rss_mib":592.97265625},{"t_s":40,"rss_mib":592.97265625},{"t_s":40,"rss_mib":592.97265625},{"t_s":41,"rss_mib":592.97265625},{"t_s":41,"rss_mib":592.97265625},{"t_s":42,"rss_mib":592.97265625},{"t_s":42,"rss_mib":592.97265625},{"t_s":43,"rss_mib":592.97265625},{"t_s":43,"rss_mib":592.97265625},{"t_s":43,"rss_mib":592.97265625},{"t_s":44,"rss_mib":592.97265625},{"t_s":45,"rss_mib":592.97265625},{"t_s":45,"rss_mib":592.97265625},{"t_s":46,"rss_mib":592.97265625},{"t_s":46,"rss_mib":592.97265625},{"t_s":47,"rss_mib":592.97265625},{"t_s":47,"rss_mib":592.97265625},{"t_s":48,"rss_mib":592.97265625},{"t_s":48,"rss_mib":592.97265625},{"t_s":49,"rss_mib":592.97265625},{"t_s":49,"rss_mib":592.97265625},{"t_s":50,"rss_mib":592.97265625},{"t_s":50,"rss_mib":592.97265625},{"t_s":51,"rss_mib":592.97265625},{"t_s":51,"rss_mib":592.97265625},{"t_s":52,"rss_mib":592.97265625},{"t_s":52,"rss_mib":592.97265625},{"t_s":53,"rss_mib":592.97265625},{"t_s":53,"rss_mib":592.97265625},{"t_s":54,"rss_mib":592.97265625},{"t_s":54,"rss_mib":592.97265625},{"t_s":55,"rss_mib":592.97265625},{"t_s":55,"rss_mib":592.97265625},{"t_s":56,"rss_mib":592.97265625},{"t_s":56,"rss_mib":592.97265625},{"t_s":57,"rss_mib":592.97265625},{"t_s":57,"rss_mib":592.97265625},{"t_s":58,"rss_mib":592.97265625},{"t_s":58,"rss_mib":592.97265625},{"t_s":59,"rss_mib":592.97265625},{"t_s":59,"rss_mib":592.97265625},{"t_s":60,"rss_mib":592.97265625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.023627381,"cost":6.136350492,"memory":436.755799466,"streaming":0.00000204,"streams_sustained":0.000001801,"throughput":485.000084531},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c2048.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"cohere":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/v2/chat","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"anthropic":{"served":true,"reason":null,"status":"","path":"/v1/messages","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":16315,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":17156,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":17184,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":18047.337278106508,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":55.40983606557377,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25012322825186667,"certified":true,"suppressed":false},"cost_window_ok":{"value":338,"certified":true,"suppressed":false},"cost_window_rps":{"value":56,"certified":true,"suppressed":false},"cost_threads":{"value":17,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.07396449704142012,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":62,"certified":true,"suppressed":false},"concurrency":2,"p99_us":34711,"first_disqualified_conc":4,"lower_bound":false,"samples":376},{"bound_ms":100,"rps":{"value":62,"certified":true,"suppressed":false},"concurrency":2,"p99_us":34711,"first_disqualified_conc":8,"lower_bound":false,"samples":376},{"bound_ms":null,"rps":{"value":63,"certified":true,"suppressed":false},"concurrency":64,"p99_us":1110053,"first_disqualified_conc":4096,"lower_bound":false,"samples":382}],"sweep":[{"conc":1,"ok":371,"rps":61,"p99_us":16972,"fail":0},{"conc":1,"ok":371,"rps":61,"p99_us":16982,"fail":0},{"conc":1,"ok":370,"rps":61,"p99_us":16988,"fail":0},{"conc":2,"ok":376,"rps":62,"p99_us":34711,"fail":0},{"conc":2,"ok":373,"rps":61,"p99_us":33315,"fail":0},{"conc":2,"ok":346,"rps":57,"p99_us":34436,"fail":0},{"conc":4,"ok":372,"rps":61,"p99_us":68169,"fail":0},{"conc":4,"ok":375,"rps":61,"p99_us":66492,"fail":0},{"conc":4,"ok":371,"rps":61,"p99_us":67604,"fail":0},{"conc":8,"ok":370,"rps":61,"p99_us":137873,"fail":0},{"conc":8,"ok":372,"rps":61,"p99_us":135334,"fail":0},{"conc":8,"ok":370,"rps":61,"p99_us":138613,"fail":0},{"conc":16,"ok":369,"rps":61,"p99_us":275281,"fail":0},{"conc":16,"ok":369,"rps":61,"p99_us":279676,"fail":0},{"conc":16,"ok":369,"rps":61,"p99_us":274766,"fail":0},{"conc":32,"ok":368,"rps":61,"p99_us":756263,"fail":0},{"conc":32,"ok":366,"rps":60,"p99_us":784753,"fail":0},{"conc":32,"ok":365,"rps":60,"p99_us":788448,"fail":0},{"conc":64,"ok":357,"rps":59,"p99_us":2426609,"fail":0},{"conc":64,"ok":382,"rps":63,"p99_us":1110053,"fail":0},{"conc":64,"ok":369,"rps":61,"p99_us":1586598,"fail":0},{"conc":128,"ok":324,"rps":52,"p99_us":2553967,"fail":0},{"conc":128,"ok":340,"rps":56,"p99_us":5719996,"fail":0},{"conc":128,"ok":340,"rps":56,"p99_us":5506700,"fail":0},{"conc":256,"ok":321,"rps":53,"p99_us":5608730,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":3843301,"fail":0},{"conc":256,"ok":328,"rps":54,"p99_us":5683967,"fail":0},{"conc":512,"ok":288,"rps":47,"p99_us":5458307,"fail":0},{"conc":512,"ok":324,"rps":52,"p99_us":5136534,"fail":0},{"conc":512,"ok":341,"rps":56,"p99_us":5576053,"fail":0},{"conc":1024,"ok":335,"rps":55,"p99_us":5530281,"fail":0},{"conc":1024,"ok":339,"rps":56,"p99_us":5875712,"fail":0},{"conc":1024,"ok":341,"rps":56,"p99_us":5761126,"fail":0},{"conc":2048,"ok":359,"rps":59,"p99_us":5857050,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":16982,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":34436,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":67604,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":137873,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":275281,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":784753,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":1586598,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":5506700,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":53,"certified":true,"suppressed":false},"p99_us":5608730,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":52,"certified":true,"suppressed":false},"p99_us":5458307,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":5761126,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":5857050,"windows":3,"clean_windows":1},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 0 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 366 successful gateway round trip(s) and 262394 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":366,"direct_c1_samples":262394},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.72265625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":687.4609375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":305,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.72265625},{"t_s":2,"rss_mib":686.45703125},{"t_s":2,"rss_mib":686.45703125},{"t_s":5,"rss_mib":686.609375},{"t_s":6,"rss_mib":686.6171875},{"t_s":8,"rss_mib":686.76171875},{"t_s":9,"rss_mib":686.76953125},{"t_s":11,"rss_mib":686.890625},{"t_s":12,"rss_mib":686.90234375},{"t_s":15,"rss_mib":686.99609375},{"t_s":15,"rss_mib":687.00390625},{"t_s":17,"rss_mib":687.125},{"t_s":18,"rss_mib":687.125},{"t_s":20,"rss_mib":687.1796875},{"t_s":21,"rss_mib":687.1796875},{"t_s":23,"rss_mib":687.21484375},{"t_s":24,"rss_mib":687.21484375},{"t_s":27,"rss_mib":687.27734375},{"t_s":27,"rss_mib":687.27734375},{"t_s":30,"rss_mib":687.32421875},{"t_s":30,"rss_mib":687.32421875},{"t_s":33,"rss_mib":687.359375},{"t_s":33,"rss_mib":687.359375},{"t_s":36,"rss_mib":687.359375},{"t_s":37,"rss_mib":687.37109375},{"t_s":39,"rss_mib":687.37109375},{"t_s":42,"rss_mib":687.37109375},{"t_s":45,"rss_mib":687.375},{"t_s":45,"rss_mib":687.375},{"t_s":46,"rss_mib":687.3828125},{"t_s":48,"rss_mib":687.3828125},{"t_s":50,"rss_mib":687.38671875},{"t_s":51,"rss_mib":687.38671875},{"t_s":53,"rss_mib":687.39453125},{"t_s":54,"rss_mib":687.39453125},{"t_s":57,"rss_mib":687.39453125},{"t_s":58,"rss_mib":687.3984375},{"t_s":61,"rss_mib":687.3984375},{"t_s":63,"rss_mib":687.3984375},{"t_s":67,"rss_mib":687.3984375},{"t_s":70,"rss_mib":687.3984375},{"t_s":72,"rss_mib":687.41015625},{"t_s":73,"rss_mib":687.41015625},{"t_s":73,"rss_mib":687.41796875},{"t_s":76,"rss_mib":687.41796875},{"t_s":78,"rss_mib":687.421875},{"t_s":79,"rss_mib":687.421875},{"t_s":82,"rss_mib":687.421875},{"t_s":82,"rss_mib":687.42578125},{"t_s":85,"rss_mib":687.42578125},{"t_s":88,"rss_mib":687.42578125},{"t_s":91,"rss_mib":687.42578125},{"t_s":94,"rss_mib":687.42578125},{"t_s":97,"rss_mib":687.42578125},{"t_s":99,"rss_mib":687.43359375},{"t_s":100,"rss_mib":687.43359375},{"t_s":102,"rss_mib":687.4453125},{"t_s":103,"rss_mib":687.4453125},{"t_s":106,"rss_mib":687.4453125},{"t_s":109,"rss_mib":687.44921875},{"t_s":109,"rss_mib":687.44921875},{"t_s":112,"rss_mib":687.44921875},{"t_s":114,"rss_mib":687.4609375},{"t_s":115,"rss_mib":687.4609375},{"t_s":118,"rss_mib":687.4609375},{"t_s":122,"rss_mib":687.4609375},{"t_s":124,"rss_mib":687.4609375},{"t_s":128,"rss_mib":687.4609375},{"t_s":131,"rss_mib":687.4609375},{"t_s":134,"rss_mib":687.4609375},{"t_s":137,"rss_mib":687.4609375},{"t_s":140,"rss_mib":687.4609375},{"t_s":143,"rss_mib":687.4609375},{"t_s":146,"rss_mib":687.4609375},{"t_s":149,"rss_mib":687.4609375},{"t_s":152,"rss_mib":687.4609375},{"t_s":155,"rss_mib":687.4609375},{"t_s":158,"rss_mib":687.4609375},{"t_s":161,"rss_mib":687.4609375},{"t_s":164,"rss_mib":687.4609375},{"t_s":167,"rss_mib":687.4609375},{"t_s":170,"rss_mib":687.4609375},{"t_s":173,"rss_mib":687.4609375},{"t_s":176,"rss_mib":687.4609375},{"t_s":179,"rss_mib":687.4609375},{"t_s":182,"rss_mib":687.4609375},{"t_s":185,"rss_mib":687.4609375},{"t_s":189,"rss_mib":687.4609375},{"t_s":192,"rss_mib":687.4609375},{"t_s":195,"rss_mib":687.4609375},{"t_s":198,"rss_mib":687.4609375},{"t_s":201,"rss_mib":687.4609375},{"t_s":204,"rss_mib":687.4609375},{"t_s":207,"rss_mib":687.4609375},{"t_s":210,"rss_mib":687.4609375},{"t_s":213,"rss_mib":687.4609375},{"t_s":216,"rss_mib":687.4609375},{"t_s":219,"rss_mib":687.4609375},{"t_s":222,"rss_mib":687.4609375},{"t_s":225,"rss_mib":687.4609375},{"t_s":228,"rss_mib":687.4609375},{"t_s":231,"rss_mib":687.4609375},{"t_s":234,"rss_mib":687.4609375},{"t_s":237,"rss_mib":687.4609375},{"t_s":240,"rss_mib":687.4609375},{"t_s":243,"rss_mib":687.4609375},{"t_s":246,"rss_mib":687.4609375},{"t_s":250,"rss_mib":687.4609375},{"t_s":253,"rss_mib":687.4609375},{"t_s":256,"rss_mib":687.4609375},{"t_s":259,"rss_mib":687.4609375},{"t_s":262,"rss_mib":687.4609375},{"t_s":265,"rss_mib":687.4609375},{"t_s":268,"rss_mib":687.4609375},{"t_s":271,"rss_mib":687.4609375},{"t_s":274,"rss_mib":687.4609375},{"t_s":277,"rss_mib":687.4609375},{"t_s":280,"rss_mib":687.4609375},{"t_s":283,"rss_mib":687.4609375},{"t_s":286,"rss_mib":687.4609375},{"t_s":289,"rss_mib":687.4609375},{"t_s":292,"rss_mib":687.4609375},{"t_s":295,"rss_mib":687.4609375},{"t_s":298,"rss_mib":687.4609375},{"t_s":301,"rss_mib":687.4609375},{"t_s":304,"rss_mib":687.4609375},{"t_s":307,"rss_mib":687.4609375},{"t_s":310,"rss_mib":687.4609375},{"t_s":314,"rss_mib":687.4609375},{"t_s":317,"rss_mib":687.4609375},{"t_s":320,"rss_mib":687.4609375},{"t_s":323,"rss_mib":687.4609375},{"t_s":326,"rss_mib":687.4609375},{"t_s":329,"rss_mib":687.4609375},{"t_s":332,"rss_mib":687.4609375},{"t_s":335,"rss_mib":687.4609375},{"t_s":338,"rss_mib":687.4609375},{"t_s":341,"rss_mib":687.4609375},{"t_s":344,"rss_mib":687.4609375},{"t_s":347,"rss_mib":687.4609375},{"t_s":350,"rss_mib":687.4609375},{"t_s":353,"rss_mib":687.4609375},{"t_s":356,"rss_mib":687.4609375},{"t_s":359,"rss_mib":687.4609375},{"t_s":362,"rss_mib":687.4609375},{"t_s":365,"rss_mib":687.4609375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.72265625},{"t_s":0,"rss_mib":592.72265625},{"t_s":0,"rss_mib":592.72265625},{"t_s":1,"rss_mib":592.72265625},{"t_s":1,"rss_mib":592.72265625},{"t_s":2,"rss_mib":592.72265625},{"t_s":2,"rss_mib":592.72265625},{"t_s":3,"rss_mib":592.72265625},{"t_s":3,"rss_mib":592.72265625},{"t_s":4,"rss_mib":592.72265625},{"t_s":5,"rss_mib":592.72265625},{"t_s":5,"rss_mib":592.72265625},{"t_s":5,"rss_mib":592.72265625},{"t_s":6,"rss_mib":592.72265625},{"t_s":6,"rss_mib":592.72265625},{"t_s":7,"rss_mib":592.72265625},{"t_s":7,"rss_mib":592.72265625},{"t_s":8,"rss_mib":592.72265625},{"t_s":8,"rss_mib":592.72265625},{"t_s":9,"rss_mib":592.72265625},{"t_s":10,"rss_mib":592.72265625},{"t_s":10,"rss_mib":592.72265625},{"t_s":10,"rss_mib":592.72265625},{"t_s":11,"rss_mib":592.72265625},{"t_s":11,"rss_mib":592.72265625},{"t_s":12,"rss_mib":592.72265625},{"t_s":12,"rss_mib":592.72265625},{"t_s":13,"rss_mib":592.72265625},{"t_s":13,"rss_mib":592.72265625},{"t_s":14,"rss_mib":592.72265625},{"t_s":15,"rss_mib":592.72265625},{"t_s":15,"rss_mib":592.72265625},{"t_s":15,"rss_mib":592.72265625},{"t_s":16,"rss_mib":592.72265625},{"t_s":16,"rss_mib":592.72265625},{"t_s":17,"rss_mib":592.72265625},{"t_s":17,"rss_mib":592.72265625},{"t_s":18,"rss_mib":592.72265625},{"t_s":18,"rss_mib":592.72265625},{"t_s":19,"rss_mib":592.72265625},{"t_s":20,"rss_mib":592.72265625},{"t_s":20,"rss_mib":592.72265625},{"t_s":20,"rss_mib":592.72265625},{"t_s":21,"rss_mib":592.72265625},{"t_s":21,"rss_mib":592.72265625},{"t_s":22,"rss_mib":592.72265625},{"t_s":22,"rss_mib":592.72265625},{"t_s":23,"rss_mib":592.72265625},{"t_s":23,"rss_mib":592.72265625},{"t_s":24,"rss_mib":592.72265625},{"t_s":25,"rss_mib":592.72265625},{"t_s":25,"rss_mib":592.72265625},{"t_s":25,"rss_mib":592.72265625},{"t_s":26,"rss_mib":592.72265625},{"t_s":26,"rss_mib":592.72265625},{"t_s":27,"rss_mib":592.72265625},{"t_s":27,"rss_mib":592.72265625},{"t_s":28,"rss_mib":592.72265625},{"t_s":28,"rss_mib":592.72265625},{"t_s":29,"rss_mib":592.72265625},{"t_s":30,"rss_mib":592.72265625},{"t_s":30,"rss_mib":592.72265625},{"t_s":30,"rss_mib":592.72265625},{"t_s":31,"rss_mib":592.72265625},{"t_s":31,"rss_mib":592.72265625},{"t_s":32,"rss_mib":592.72265625},{"t_s":32,"rss_mib":592.72265625},{"t_s":33,"rss_mib":592.72265625},{"t_s":33,"rss_mib":592.72265625},{"t_s":34,"rss_mib":592.72265625},{"t_s":35,"rss_mib":592.72265625},{"t_s":35,"rss_mib":592.72265625},{"t_s":35,"rss_mib":592.72265625},{"t_s":36,"rss_mib":592.72265625},{"t_s":36,"rss_mib":592.72265625},{"t_s":37,"rss_mib":592.72265625},{"t_s":37,"rss_mib":592.72265625},{"t_s":38,"rss_mib":592.72265625},{"t_s":38,"rss_mib":592.72265625},{"t_s":39,"rss_mib":592.72265625},{"t_s":40,"rss_mib":592.72265625},{"t_s":40,"rss_mib":592.72265625},{"t_s":40,"rss_mib":592.72265625},{"t_s":41,"rss_mib":592.72265625},{"t_s":41,"rss_mib":592.72265625},{"t_s":42,"rss_mib":592.72265625},{"t_s":42,"rss_mib":592.72265625},{"t_s":43,"rss_mib":592.72265625},{"t_s":43,"rss_mib":592.72265625},{"t_s":44,"rss_mib":592.72265625},{"t_s":45,"rss_mib":592.72265625},{"t_s":45,"rss_mib":592.72265625},{"t_s":45,"rss_mib":592.72265625},{"t_s":46,"rss_mib":592.72265625},{"t_s":46,"rss_mib":592.72265625},{"t_s":47,"rss_mib":592.72265625},{"t_s":47,"rss_mib":592.72265625},{"t_s":48,"rss_mib":592.72265625},{"t_s":49,"rss_mib":592.72265625},{"t_s":49,"rss_mib":592.72265625},{"t_s":50,"rss_mib":592.72265625},{"t_s":50,"rss_mib":592.72265625},{"t_s":50,"rss_mib":592.72265625},{"t_s":51,"rss_mib":592.72265625},{"t_s":51,"rss_mib":592.72265625},{"t_s":52,"rss_mib":592.72265625},{"t_s":52,"rss_mib":592.72265625},{"t_s":53,"rss_mib":592.72265625},{"t_s":54,"rss_mib":592.72265625},{"t_s":54,"rss_mib":592.72265625},{"t_s":55,"rss_mib":592.72265625},{"t_s":55,"rss_mib":592.72265625},{"t_s":55,"rss_mib":592.72265625},{"t_s":56,"rss_mib":592.72265625},{"t_s":56,"rss_mib":592.72265625},{"t_s":57,"rss_mib":592.72265625},{"t_s":57,"rss_mib":592.72265625},{"t_s":58,"rss_mib":592.72265625},{"t_s":59,"rss_mib":592.72265625},{"t_s":59,"rss_mib":592.72265625},{"t_s":59,"rss_mib":592.72265625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.010033595,"cost":6.109850332,"memory":441.311885475,"streaming":0.000002448,"streams_sustained":0.000001772,"throughput":473.122679448},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c2048.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/gpt-4o-mini-bedrock:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":21239,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":22150,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":22177,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":22086.330935251797,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":45.276872964169385,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2506164650746462,"certified":true,"suppressed":false},"cost_window_ok":{"value":278,"certified":true,"suppressed":false},"cost_window_rps":{"value":46,"certified":true,"suppressed":false},"cost_threads":{"value":17,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.07913669064748201,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":46,"certified":true,"suppressed":false},"concurrency":1,"p99_us":22251,"first_disqualified_conc":4,"lower_bound":false,"samples":280},{"bound_ms":100,"rps":{"value":46,"certified":true,"suppressed":false},"concurrency":1,"p99_us":22251,"first_disqualified_conc":8,"lower_bound":false,"samples":280},{"bound_ms":null,"rps":{"value":46,"certified":true,"suppressed":false},"concurrency":1,"p99_us":22251,"first_disqualified_conc":4096,"lower_bound":false,"samples":280}],"sweep":[{"conc":1,"ok":280,"rps":46,"p99_us":22303,"fail":0},{"conc":1,"ok":280,"rps":46,"p99_us":22254,"fail":0},{"conc":1,"ok":280,"rps":46,"p99_us":22251,"fail":0},{"conc":2,"ok":281,"rps":46,"p99_us":43995,"fail":0},{"conc":2,"ok":279,"rps":46,"p99_us":47036,"fail":0},{"conc":2,"ok":280,"rps":46,"p99_us":43476,"fail":0},{"conc":4,"ok":280,"rps":46,"p99_us":90344,"fail":0},{"conc":4,"ok":278,"rps":46,"p99_us":91342,"fail":0},{"conc":4,"ok":280,"rps":46,"p99_us":92872,"fail":0},{"conc":8,"ok":278,"rps":46,"p99_us":180677,"fail":0},{"conc":8,"ok":278,"rps":46,"p99_us":181255,"fail":0},{"conc":8,"ok":277,"rps":46,"p99_us":180820,"fail":0},{"conc":16,"ok":276,"rps":45,"p99_us":368035,"fail":0},{"conc":16,"ok":277,"rps":46,"p99_us":402679,"fail":0},{"conc":16,"ok":275,"rps":45,"p99_us":398164,"fail":0},{"conc":32,"ok":270,"rps":44,"p99_us":1087125,"fail":0},{"conc":32,"ok":272,"rps":45,"p99_us":1070728,"fail":0},{"conc":32,"ok":273,"rps":45,"p99_us":1088478,"fail":0},{"conc":64,"ok":264,"rps":43,"p99_us":3356079,"fail":0},{"conc":64,"ok":263,"rps":43,"p99_us":3451532,"fail":0},{"conc":64,"ok":263,"rps":43,"p99_us":3368274,"fail":0},{"conc":128,"ok":231,"rps":38,"p99_us":5672236,"fail":0},{"conc":128,"ok":256,"rps":42,"p99_us":2741112,"fail":0},{"conc":128,"ok":243,"rps":40,"p99_us":5690049,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":5406286,"fail":0},{"conc":256,"ok":205,"rps":34,"p99_us":5596347,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":5353927,"fail":0},{"conc":512,"ok":210,"rps":34,"p99_us":5781393,"fail":0},{"conc":512,"ok":179,"rps":29,"p99_us":5732012,"fail":0},{"conc":512,"ok":254,"rps":42,"p99_us":5737865,"fail":0},{"conc":1024,"ok":161,"rps":26,"p99_us":5768637,"fail":0},{"conc":1024,"ok":258,"rps":42,"p99_us":5829111,"fail":0},{"conc":1024,"ok":173,"rps":28,"p99_us":5290450,"fail":0},{"conc":2048,"ok":136,"rps":22,"p99_us":5676636,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":391},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":527},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":46,"certified":true,"suppressed":false},"p99_us":22254,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":46,"certified":true,"suppressed":false},"p99_us":43995,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":46,"certified":true,"suppressed":false},"p99_us":91342,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":46,"certified":true,"suppressed":false},"p99_us":180820,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":45,"certified":true,"suppressed":false},"p99_us":398164,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":45,"certified":true,"suppressed":false},"p99_us":1087125,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":43,"certified":true,"suppressed":false},"p99_us":3368274,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":40,"certified":true,"suppressed":false},"p99_us":5672236,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":42,"certified":true,"suppressed":false},"p99_us":5406286,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":34,"certified":true,"suppressed":false},"p99_us":5737865,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":28,"certified":true,"suppressed":false},"p99_us":5768637,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":22,"certified":true,"suppressed":false},"p99_us":5676636,"windows":3,"clean_windows":1},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 281 successful gateway round trip(s) and 267100 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":281,"direct_c1_samples":267100},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.921875,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":692.7734375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":692.77734375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":692.77734375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":692.77734375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":64,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.005149277000641089,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.921875},{"t_s":2,"rss_mib":687.07421875},{"t_s":2,"rss_mib":687.078125},{"t_s":5,"rss_mib":687.51171875},{"t_s":6,"rss_mib":687.51171875},{"t_s":8,"rss_mib":687.55859375},{"t_s":8,"rss_mib":687.55859375},{"t_s":11,"rss_mib":688.23046875},{"t_s":12,"rss_mib":688.23046875},{"t_s":14,"rss_mib":688.39453125},{"t_s":15,"rss_mib":688.39453125},{"t_s":17,"rss_mib":688.62890625},{"t_s":18,"rss_mib":688.64453125},{"t_s":21,"rss_mib":689.08203125},{"t_s":21,"rss_mib":689.08203125},{"t_s":22,"rss_mib":689.109375},{"t_s":24,"rss_mib":689.109375},{"t_s":24,"rss_mib":689.16796875},{"t_s":27,"rss_mib":689.16796875},{"t_s":28,"rss_mib":689.6171875},{"t_s":30,"rss_mib":689.6171875},{"t_s":31,"rss_mib":689.81640625},{"t_s":33,"rss_mib":689.81640625},{"t_s":35,"rss_mib":689.8828125},{"t_s":36,"rss_mib":689.8828125},{"t_s":38,"rss_mib":690.1171875},{"t_s":39,"rss_mib":690.1171875},{"t_s":41,"rss_mib":690.3515625},{"t_s":42,"rss_mib":690.3515625},{"t_s":45,"rss_mib":690.3828125},{"t_s":45,"rss_mib":690.3828125},{"t_s":47,"rss_mib":690.515625},{"t_s":48,"rss_mib":690.515625},{"t_s":50,"rss_mib":690.73046875},{"t_s":51,"rss_mib":690.73046875},{"t_s":54,"rss_mib":690.73046875},{"t_s":54,"rss_mib":690.734375},{"t_s":57,"rss_mib":690.734375},{"t_s":60,"rss_mib":690.734375},{"t_s":63,"rss_mib":690.734375},{"t_s":66,"rss_mib":690.734375},{"t_s":69,"rss_mib":690.734375},{"t_s":70,"rss_mib":690.73828125},{"t_s":72,"rss_mib":690.73828125},{"t_s":75,"rss_mib":690.73828125},{"t_s":77,"rss_mib":690.7421875},{"t_s":78,"rss_mib":690.7421875},{"t_s":81,"rss_mib":690.7421875},{"t_s":84,"rss_mib":690.7421875},{"t_s":87,"rss_mib":690.7421875},{"t_s":90,"rss_mib":690.7421875},{"t_s":93,"rss_mib":690.7421875},{"t_s":96,"rss_mib":690.78125},{"t_s":96,"rss_mib":690.78125},{"t_s":99,"rss_mib":690.78125},{"t_s":102,"rss_mib":690.78125},{"t_s":105,"rss_mib":690.78125},{"t_s":108,"rss_mib":690.84375},{"t_s":108,"rss_mib":690.90625},{"t_s":111,"rss_mib":690.90625},{"t_s":114,"rss_mib":691.29296875},{"t_s":114,"rss_mib":691.30078125},{"t_s":117,"rss_mib":691.30078125},{"t_s":119,"rss_mib":691.59765625},{"t_s":120,"rss_mib":691.59765625},{"t_s":121,"rss_mib":691.62109375},{"t_s":123,"rss_mib":691.62109375},{"t_s":125,"rss_mib":692.03515625},{"t_s":127,"rss_mib":692.03515625},{"t_s":128,"rss_mib":692.13671875},{"t_s":130,"rss_mib":692.13671875},{"t_s":132,"rss_mib":692.40234375},{"t_s":133,"rss_mib":692.40234375},{"t_s":134,"rss_mib":692.57421875},{"t_s":136,"rss_mib":692.57421875},{"t_s":139,"rss_mib":692.57421875},{"t_s":142,"rss_mib":692.57421875},{"t_s":145,"rss_mib":692.57421875},{"t_s":148,"rss_mib":692.57421875},{"t_s":151,"rss_mib":692.57421875},{"t_s":154,"rss_mib":692.57421875},{"t_s":157,"rss_mib":692.57421875},{"t_s":160,"rss_mib":692.57421875},{"t_s":163,"rss_mib":692.57421875},{"t_s":166,"rss_mib":692.57421875},{"t_s":169,"rss_mib":692.57421875},{"t_s":172,"rss_mib":692.57421875},{"t_s":175,"rss_mib":692.57421875},{"t_s":178,"rss_mib":692.57421875},{"t_s":181,"rss_mib":692.57421875},{"t_s":184,"rss_mib":692.57421875},{"t_s":187,"rss_mib":692.57421875},{"t_s":188,"rss_mib":692.578125},{"t_s":190,"rss_mib":692.578125},{"t_s":190,"rss_mib":692.58984375},{"t_s":193,"rss_mib":692.58984375},{"t_s":196,"rss_mib":692.58984375},{"t_s":197,"rss_mib":692.59765625},{"t_s":199,"rss_mib":692.59765625},{"t_s":201,"rss_mib":692.6015625},{"t_s":202,"rss_mib":692.6015625},{"t_s":205,"rss_mib":692.6015625},{"t_s":208,"rss_mib":692.6015625},{"t_s":211,"rss_mib":692.6015625},{"t_s":214,"rss_mib":692.60546875},{"t_s":216,"rss_mib":692.7734375},{"t_s":217,"rss_mib":692.7734375},{"t_s":220,"rss_mib":692.7734375},{"t_s":223,"rss_mib":692.7734375},{"t_s":226,"rss_mib":692.7734375},{"t_s":229,"rss_mib":692.7734375},{"t_s":232,"rss_mib":692.7734375},{"t_s":235,"rss_mib":692.7734375},{"t_s":239,"rss_mib":692.7734375},{"t_s":241,"rss_mib":692.7734375},{"t_s":245,"rss_mib":692.7734375},{"t_s":247,"rss_mib":692.7734375},{"t_s":251,"rss_mib":692.7734375},{"t_s":254,"rss_mib":692.7734375},{"t_s":257,"rss_mib":692.7734375},{"t_s":260,"rss_mib":692.7734375},{"t_s":262,"rss_mib":692.77734375},{"t_s":263,"rss_mib":692.77734375},{"t_s":266,"rss_mib":692.77734375},{"t_s":269,"rss_mib":692.77734375},{"t_s":272,"rss_mib":692.77734375},{"t_s":275,"rss_mib":692.77734375},{"t_s":278,"rss_mib":692.77734375},{"t_s":281,"rss_mib":692.77734375},{"t_s":284,"rss_mib":692.77734375},{"t_s":287,"rss_mib":692.77734375},{"t_s":290,"rss_mib":692.77734375},{"t_s":293,"rss_mib":692.77734375},{"t_s":296,"rss_mib":692.77734375},{"t_s":299,"rss_mib":692.77734375},{"t_s":302,"rss_mib":692.77734375},{"t_s":305,"rss_mib":692.77734375},{"t_s":308,"rss_mib":692.77734375},{"t_s":311,"rss_mib":692.77734375},{"t_s":314,"rss_mib":692.77734375},{"t_s":317,"rss_mib":692.77734375},{"t_s":320,"rss_mib":692.77734375},{"t_s":323,"rss_mib":692.77734375},{"t_s":326,"rss_mib":692.77734375},{"t_s":329,"rss_mib":692.77734375},{"t_s":332,"rss_mib":692.77734375},{"t_s":335,"rss_mib":692.77734375},{"t_s":338,"rss_mib":692.77734375},{"t_s":341,"rss_mib":692.77734375},{"t_s":344,"rss_mib":692.77734375},{"t_s":347,"rss_mib":692.77734375},{"t_s":350,"rss_mib":692.77734375},{"t_s":353,"rss_mib":692.77734375},{"t_s":356,"rss_mib":692.77734375},{"t_s":359,"rss_mib":692.77734375},{"t_s":362,"rss_mib":692.77734375}],"idle_rss_series":[{"t_s":0,"rss_mib":591.921875},{"t_s":0,"rss_mib":591.921875},{"t_s":0,"rss_mib":591.921875},{"t_s":1,"rss_mib":591.921875},{"t_s":1,"rss_mib":591.921875},{"t_s":2,"rss_mib":591.921875},{"t_s":2,"rss_mib":591.921875},{"t_s":3,"rss_mib":591.921875},{"t_s":3,"rss_mib":591.921875},{"t_s":4,"rss_mib":591.921875},{"t_s":5,"rss_mib":591.921875},{"t_s":5,"rss_mib":591.921875},{"t_s":5,"rss_mib":591.921875},{"t_s":6,"rss_mib":591.921875},{"t_s":6,"rss_mib":591.921875},{"t_s":7,"rss_mib":591.921875},{"t_s":7,"rss_mib":591.921875},{"t_s":8,"rss_mib":591.921875},{"t_s":8,"rss_mib":591.921875},{"t_s":9,"rss_mib":591.921875},{"t_s":10,"rss_mib":591.921875},{"t_s":10,"rss_mib":591.921875},{"t_s":10,"rss_mib":591.921875},{"t_s":11,"rss_mib":591.921875},{"t_s":11,"rss_mib":591.921875},{"t_s":12,"rss_mib":591.921875},{"t_s":12,"rss_mib":591.921875},{"t_s":13,"rss_mib":591.921875},{"t_s":13,"rss_mib":591.921875},{"t_s":14,"rss_mib":591.921875},{"t_s":15,"rss_mib":591.921875},{"t_s":15,"rss_mib":591.921875},{"t_s":15,"rss_mib":591.921875},{"t_s":16,"rss_mib":591.921875},{"t_s":16,"rss_mib":591.921875},{"t_s":17,"rss_mib":591.921875},{"t_s":17,"rss_mib":591.921875},{"t_s":18,"rss_mib":591.921875},{"t_s":18,"rss_mib":591.921875},{"t_s":19,"rss_mib":591.921875},{"t_s":20,"rss_mib":591.921875},{"t_s":20,"rss_mib":591.921875},{"t_s":20,"rss_mib":591.921875},{"t_s":21,"rss_mib":591.921875},{"t_s":21,"rss_mib":591.921875},{"t_s":22,"rss_mib":591.921875},{"t_s":22,"rss_mib":591.921875},{"t_s":23,"rss_mib":591.921875},{"t_s":23,"rss_mib":591.921875},{"t_s":24,"rss_mib":591.921875},{"t_s":25,"rss_mib":591.921875},{"t_s":25,"rss_mib":591.921875},{"t_s":25,"rss_mib":591.921875},{"t_s":26,"rss_mib":591.921875},{"t_s":26,"rss_mib":591.921875},{"t_s":27,"rss_mib":591.921875},{"t_s":27,"rss_mib":591.921875},{"t_s":28,"rss_mib":591.921875},{"t_s":28,"rss_mib":591.921875},{"t_s":29,"rss_mib":591.921875},{"t_s":30,"rss_mib":591.921875},{"t_s":30,"rss_mib":591.921875},{"t_s":30,"rss_mib":591.921875},{"t_s":31,"rss_mib":591.921875},{"t_s":31,"rss_mib":591.921875},{"t_s":32,"rss_mib":591.921875},{"t_s":32,"rss_mib":591.921875},{"t_s":33,"rss_mib":591.921875},{"t_s":33,"rss_mib":591.921875},{"t_s":34,"rss_mib":591.921875},{"t_s":35,"rss_mib":591.921875},{"t_s":35,"rss_mib":591.921875},{"t_s":35,"rss_mib":591.921875},{"t_s":36,"rss_mib":591.921875},{"t_s":36,"rss_mib":591.921875},{"t_s":37,"rss_mib":591.921875},{"t_s":37,"rss_mib":591.921875},{"t_s":38,"rss_mib":591.921875},{"t_s":38,"rss_mib":591.921875},{"t_s":39,"rss_mib":591.921875},{"t_s":40,"rss_mib":591.921875},{"t_s":40,"rss_mib":591.921875},{"t_s":40,"rss_mib":591.921875},{"t_s":41,"rss_mib":591.921875},{"t_s":41,"rss_mib":591.921875},{"t_s":42,"rss_mib":591.921875},{"t_s":42,"rss_mib":591.921875},{"t_s":43,"rss_mib":591.921875},{"t_s":43,"rss_mib":591.921875},{"t_s":44,"rss_mib":591.921875},{"t_s":45,"rss_mib":591.921875},{"t_s":45,"rss_mib":591.921875},{"t_s":45,"rss_mib":591.921875},{"t_s":46,"rss_mib":591.921875},{"t_s":46,"rss_mib":591.921875},{"t_s":47,"rss_mib":591.921875},{"t_s":47,"rss_mib":591.921875},{"t_s":48,"rss_mib":591.921875},{"t_s":48,"rss_mib":591.921875},{"t_s":49,"rss_mib":591.921875},{"t_s":50,"rss_mib":591.921875},{"t_s":50,"rss_mib":591.921875},{"t_s":50,"rss_mib":591.921875},{"t_s":51,"rss_mib":591.921875},{"t_s":51,"rss_mib":591.921875},{"t_s":52,"rss_mib":591.921875},{"t_s":52,"rss_mib":591.921875},{"t_s":53,"rss_mib":591.921875},{"t_s":53,"rss_mib":591.921875},{"t_s":54,"rss_mib":591.921875},{"t_s":55,"rss_mib":591.921875},{"t_s":55,"rss_mib":591.921875},{"t_s":55,"rss_mib":591.921875},{"t_s":56,"rss_mib":591.921875},{"t_s":56,"rss_mib":591.921875},{"t_s":57,"rss_mib":591.921875},{"t_s":57,"rss_mib":591.921875},{"t_s":58,"rss_mib":591.921875},{"t_s":58,"rss_mib":591.921875},{"t_s":59,"rss_mib":591.921875},{"t_s":59,"rss_mib":591.921875}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.028735419,"cost":6.138166437,"memory":438.402770567,"streaming":0.000001549,"streams_sustained":0.00000173,"throughput":511.634200135},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c2048.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}},"bedrock":{"served":true,"reason":null,"status":"","path":"/bedrock/model/gpt-4o-mini-bedrock/converse","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10032,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":10847,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":10874,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10134.003350083753,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":98.67768595041322,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.24972426540447598,"certified":true,"suppressed":false},"cost_window_ok":{"value":597,"certified":true,"suppressed":false},"cost_window_rps":{"value":99,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.04522613065326633,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":101,"certified":true,"suppressed":false},"concurrency":2,"p99_us":20685,"first_disqualified_conc":8,"lower_bound":false,"samples":607},{"bound_ms":100,"rps":{"value":101,"certified":true,"suppressed":false},"concurrency":2,"p99_us":20685,"first_disqualified_conc":16,"lower_bound":false,"samples":607},{"bound_ms":null,"rps":{"value":104,"certified":true,"suppressed":false},"concurrency":128,"p99_us":1374776,"first_disqualified_conc":16384,"lower_bound":false,"samples":625}],"sweep":[{"conc":1,"ok":601,"rps":100,"p99_us":10639,"fail":0},{"conc":1,"ok":603,"rps":100,"p99_us":10593,"fail":0},{"conc":1,"ok":601,"rps":100,"p99_us":10681,"fail":0},{"conc":2,"ok":607,"rps":101,"p99_us":20685,"fail":0},{"conc":2,"ok":606,"rps":100,"p99_us":20712,"fail":0},{"conc":2,"ok":604,"rps":100,"p99_us":20730,"fail":0},{"conc":4,"ok":604,"rps":100,"p99_us":40962,"fail":0},{"conc":4,"ok":603,"rps":100,"p99_us":42264,"fail":0},{"conc":4,"ok":604,"rps":100,"p99_us":40761,"fail":0},{"conc":8,"ok":606,"rps":100,"p99_us":84954,"fail":0},{"conc":8,"ok":606,"rps":100,"p99_us":85491,"fail":0},{"conc":8,"ok":604,"rps":100,"p99_us":84334,"fail":0},{"conc":16,"ok":606,"rps":100,"p99_us":185377,"fail":0},{"conc":16,"ok":606,"rps":100,"p99_us":172306,"fail":0},{"conc":16,"ok":606,"rps":100,"p99_us":172612,"fail":0},{"conc":32,"ok":605,"rps":100,"p99_us":486671,"fail":0},{"conc":32,"ok":606,"rps":100,"p99_us":506824,"fail":0},{"conc":32,"ok":607,"rps":101,"p99_us":501464,"fail":0},{"conc":64,"ok":600,"rps":99,"p99_us":1748904,"fail":0},{"conc":64,"ok":597,"rps":99,"p99_us":1800431,"fail":0},{"conc":64,"ok":598,"rps":99,"p99_us":1765947,"fail":0},{"conc":128,"ok":526,"rps":87,"p99_us":5551477,"fail":0},{"conc":128,"ok":625,"rps":104,"p99_us":1374776,"fail":0},{"conc":128,"ok":514,"rps":85,"p99_us":5577364,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":2575123,"fail":0},{"conc":256,"ok":515,"rps":85,"p99_us":5464620,"fail":0},{"conc":256,"ok":576,"rps":95,"p99_us":5452543,"fail":0},{"conc":512,"ok":573,"rps":95,"p99_us":5375557,"fail":0},{"conc":512,"ok":534,"rps":88,"p99_us":5528043,"fail":0},{"conc":512,"ok":504,"rps":83,"p99_us":4862132,"fail":0},{"conc":1024,"ok":581,"rps":96,"p99_us":5511670,"fail":0},{"conc":1024,"ok":522,"rps":86,"p99_us":5564586,"fail":0},{"conc":1024,"ok":462,"rps":76,"p99_us":4892670,"fail":0},{"conc":2048,"ok":376,"rps":62,"p99_us":4365032,"fail":0},{"conc":2048,"ok":584,"rps":97,"p99_us":5483522,"fail":0},{"conc":2048,"ok":381,"rps":63,"p99_us":4896474,"fail":0},{"conc":4096,"ok":364,"rps":60,"p99_us":5438830,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":1024},{"conc":8192,"ok":36,"rps":5,"p99_us":5906902,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":2291},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":16384,"ok":1,"rps":0.1666287234734158,"p99_us":5451337,"fail":10347},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":418},{"conc":16384,"ok":373,"rps":61,"p99_us":4101393,"fail":4906}],"rungs":[{"conc":1,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":10639,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":20712,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":40962,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":84954,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":172612,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":501464,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":99,"certified":true,"suppressed":false},"p99_us":1765947,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":5551477,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5452543,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":88,"certified":true,"suppressed":false},"p99_us":5375557,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":86,"certified":true,"suppressed":false},"p99_us":5511670,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":4896474,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":5438830,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":5,"certified":true,"suppressed":false},"p99_us":5906902,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"bedrock>bedrock is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 595 successful gateway round trip(s) and 267885 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":595,"direct_c1_samples":267885},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.5078125,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":671.40234375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":303,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.5078125},{"t_s":2,"rss_mib":669.7265625},{"t_s":2,"rss_mib":669.7265625},{"t_s":3,"rss_mib":669.73828125},{"t_s":6,"rss_mib":669.73828125},{"t_s":6,"rss_mib":669.74609375},{"t_s":8,"rss_mib":669.74609375},{"t_s":12,"rss_mib":669.74609375},{"t_s":15,"rss_mib":669.74609375},{"t_s":18,"rss_mib":669.74609375},{"t_s":21,"rss_mib":669.74609375},{"t_s":24,"rss_mib":669.74609375},{"t_s":27,"rss_mib":669.74609375},{"t_s":30,"rss_mib":669.74609375},{"t_s":33,"rss_mib":669.74609375},{"t_s":36,"rss_mib":669.74609375},{"t_s":39,"rss_mib":669.74609375},{"t_s":42,"rss_mib":669.74609375},{"t_s":45,"rss_mib":669.74609375},{"t_s":48,"rss_mib":669.74609375},{"t_s":51,"rss_mib":669.74609375},{"t_s":54,"rss_mib":669.74609375},{"t_s":55,"rss_mib":669.75390625},{"t_s":57,"rss_mib":669.75390625},{"t_s":60,"rss_mib":669.75390625},{"t_s":61,"rss_mib":669.87890625},{"t_s":63,"rss_mib":669.87890625},{"t_s":66,"rss_mib":669.87890625},{"t_s":68,"rss_mib":669.8828125},{"t_s":69,"rss_mib":669.8828125},{"t_s":71,"rss_mib":669.94921875},{"t_s":72,"rss_mib":669.94921875},{"t_s":73,"rss_mib":669.953125},{"t_s":75,"rss_mib":669.953125},{"t_s":77,"rss_mib":670.03125},{"t_s":78,"rss_mib":670.03125},{"t_s":79,"rss_mib":670.03515625},{"t_s":81,"rss_mib":670.03515625},{"t_s":83,"rss_mib":670.2265625},{"t_s":84,"rss_mib":670.2265625},{"t_s":87,"rss_mib":670.2265625},{"t_s":90,"rss_mib":670.2265625},{"t_s":93,"rss_mib":670.2265625},{"t_s":96,"rss_mib":670.2265625},{"t_s":99,"rss_mib":670.2265625},{"t_s":102,"rss_mib":670.2265625},{"t_s":105,"rss_mib":670.2265625},{"t_s":108,"rss_mib":670.2265625},{"t_s":111,"rss_mib":670.2265625},{"t_s":115,"rss_mib":670.2265625},{"t_s":118,"rss_mib":670.2265625},{"t_s":121,"rss_mib":670.2265625},{"t_s":124,"rss_mib":670.2265625},{"t_s":127,"rss_mib":670.2265625},{"t_s":130,"rss_mib":670.2265625},{"t_s":133,"rss_mib":670.2265625},{"t_s":136,"rss_mib":670.2265625},{"t_s":139,"rss_mib":670.2265625},{"t_s":142,"rss_mib":670.2265625},{"t_s":145,"rss_mib":670.2265625},{"t_s":148,"rss_mib":670.2265625},{"t_s":151,"rss_mib":670.2265625},{"t_s":154,"rss_mib":670.2265625},{"t_s":157,"rss_mib":670.2265625},{"t_s":160,"rss_mib":670.2265625},{"t_s":163,"rss_mib":670.2265625},{"t_s":166,"rss_mib":670.2265625},{"t_s":169,"rss_mib":670.2265625},{"t_s":171,"rss_mib":670.234375},{"t_s":172,"rss_mib":670.234375},{"t_s":173,"rss_mib":670.23828125},{"t_s":175,"rss_mib":670.23828125},{"t_s":178,"rss_mib":670.23828125},{"t_s":181,"rss_mib":670.23828125},{"t_s":183,"rss_mib":670.38671875},{"t_s":184,"rss_mib":670.38671875},{"t_s":186,"rss_mib":670.390625},{"t_s":187,"rss_mib":670.390625},{"t_s":190,"rss_mib":670.65234375},{"t_s":190,"rss_mib":670.65234375},{"t_s":192,"rss_mib":670.6640625},{"t_s":193,"rss_mib":670.6640625},{"t_s":196,"rss_mib":670.88671875},{"t_s":196,"rss_mib":670.88671875},{"t_s":199,"rss_mib":670.91015625},{"t_s":199,"rss_mib":670.91015625},{"t_s":200,"rss_mib":670.9140625},{"t_s":202,"rss_mib":670.9140625},{"t_s":205,"rss_mib":670.91796875},{"t_s":205,"rss_mib":670.91796875},{"t_s":208,"rss_mib":671.05078125},{"t_s":208,"rss_mib":671.07421875},{"t_s":209,"rss_mib":671.109375},{"t_s":211,"rss_mib":671.11328125},{"t_s":214,"rss_mib":671.32421875},{"t_s":214,"rss_mib":671.32421875},{"t_s":217,"rss_mib":671.40234375},{"t_s":217,"rss_mib":671.40234375},{"t_s":220,"rss_mib":671.40234375},{"t_s":224,"rss_mib":671.40234375},{"t_s":227,"rss_mib":671.40234375},{"t_s":230,"rss_mib":671.40234375},{"t_s":233,"rss_mib":671.40234375},{"t_s":236,"rss_mib":671.40234375},{"t_s":239,"rss_mib":671.40234375},{"t_s":242,"rss_mib":671.40234375},{"t_s":245,"rss_mib":671.40234375},{"t_s":248,"rss_mib":671.40234375},{"t_s":251,"rss_mib":671.40234375},{"t_s":254,"rss_mib":671.40234375},{"t_s":257,"rss_mib":671.40234375},{"t_s":260,"rss_mib":671.40234375},{"t_s":263,"rss_mib":671.40234375},{"t_s":266,"rss_mib":671.40234375},{"t_s":269,"rss_mib":671.40234375},{"t_s":272,"rss_mib":671.40234375},{"t_s":275,"rss_mib":671.40234375},{"t_s":278,"rss_mib":671.40234375},{"t_s":281,"rss_mib":671.40234375},{"t_s":284,"rss_mib":671.40234375},{"t_s":287,"rss_mib":671.40234375},{"t_s":290,"rss_mib":671.40234375},{"t_s":293,"rss_mib":671.40234375},{"t_s":296,"rss_mib":671.40234375},{"t_s":299,"rss_mib":671.40234375},{"t_s":302,"rss_mib":671.40234375},{"t_s":305,"rss_mib":671.40234375},{"t_s":308,"rss_mib":671.40234375},{"t_s":311,"rss_mib":671.40234375},{"t_s":314,"rss_mib":671.40234375},{"t_s":317,"rss_mib":671.40234375},{"t_s":320,"rss_mib":671.40234375},{"t_s":323,"rss_mib":671.40234375},{"t_s":326,"rss_mib":671.40234375},{"t_s":329,"rss_mib":671.40234375},{"t_s":332,"rss_mib":671.40234375},{"t_s":336,"rss_mib":671.40234375},{"t_s":339,"rss_mib":671.40234375},{"t_s":342,"rss_mib":671.40234375},{"t_s":345,"rss_mib":671.40234375},{"t_s":348,"rss_mib":671.40234375},{"t_s":351,"rss_mib":671.40234375},{"t_s":354,"rss_mib":671.40234375},{"t_s":357,"rss_mib":671.40234375},{"t_s":360,"rss_mib":671.40234375},{"t_s":363,"rss_mib":671.40234375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.5078125},{"t_s":0,"rss_mib":592.5078125},{"t_s":0,"rss_mib":592.5078125},{"t_s":1,"rss_mib":592.5078125},{"t_s":1,"rss_mib":592.5078125},{"t_s":2,"rss_mib":592.5078125},{"t_s":2,"rss_mib":592.5078125},{"t_s":3,"rss_mib":592.5078125},{"t_s":3,"rss_mib":592.5078125},{"t_s":4,"rss_mib":592.5078125},{"t_s":5,"rss_mib":592.5078125},{"t_s":5,"rss_mib":592.5078125},{"t_s":5,"rss_mib":592.5078125},{"t_s":6,"rss_mib":592.5078125},{"t_s":6,"rss_mib":592.5078125},{"t_s":7,"rss_mib":592.5078125},{"t_s":7,"rss_mib":592.5078125},{"t_s":8,"rss_mib":592.5078125},{"t_s":8,"rss_mib":592.5078125},{"t_s":9,"rss_mib":592.5078125},{"t_s":10,"rss_mib":592.5078125},{"t_s":10,"rss_mib":592.5078125},{"t_s":10,"rss_mib":592.5078125},{"t_s":11,"rss_mib":592.5078125},{"t_s":11,"rss_mib":592.5078125},{"t_s":12,"rss_mib":592.5078125},{"t_s":12,"rss_mib":592.5078125},{"t_s":13,"rss_mib":592.5078125},{"t_s":13,"rss_mib":592.5078125},{"t_s":14,"rss_mib":592.5078125},{"t_s":15,"rss_mib":592.5078125},{"t_s":15,"rss_mib":592.5078125},{"t_s":15,"rss_mib":592.5078125},{"t_s":16,"rss_mib":592.5078125},{"t_s":16,"rss_mib":592.5078125},{"t_s":17,"rss_mib":592.5078125},{"t_s":17,"rss_mib":592.5078125},{"t_s":18,"rss_mib":592.5078125},{"t_s":18,"rss_mib":592.5078125},{"t_s":19,"rss_mib":592.5078125},{"t_s":20,"rss_mib":592.5078125},{"t_s":20,"rss_mib":592.5078125},{"t_s":20,"rss_mib":592.5078125},{"t_s":21,"rss_mib":592.5078125},{"t_s":21,"rss_mib":592.5078125},{"t_s":22,"rss_mib":592.5078125},{"t_s":22,"rss_mib":592.5078125},{"t_s":23,"rss_mib":592.5078125},{"t_s":23,"rss_mib":592.5078125},{"t_s":24,"rss_mib":592.5078125},{"t_s":25,"rss_mib":592.5078125},{"t_s":25,"rss_mib":592.5078125},{"t_s":25,"rss_mib":592.5078125},{"t_s":26,"rss_mib":592.5078125},{"t_s":26,"rss_mib":592.5078125},{"t_s":27,"rss_mib":592.5078125},{"t_s":27,"rss_mib":592.5078125},{"t_s":28,"rss_mib":592.5078125},{"t_s":28,"rss_mib":592.5078125},{"t_s":29,"rss_mib":592.5078125},{"t_s":30,"rss_mib":592.5078125},{"t_s":30,"rss_mib":592.5078125},{"t_s":30,"rss_mib":592.5078125},{"t_s":31,"rss_mib":592.5078125},{"t_s":31,"rss_mib":592.5078125},{"t_s":32,"rss_mib":592.5078125},{"t_s":32,"rss_mib":592.5078125},{"t_s":33,"rss_mib":592.5078125},{"t_s":33,"rss_mib":592.5078125},{"t_s":34,"rss_mib":592.5078125},{"t_s":35,"rss_mib":592.5078125},{"t_s":35,"rss_mib":592.5078125},{"t_s":35,"rss_mib":592.5078125},{"t_s":36,"rss_mib":592.5078125},{"t_s":36,"rss_mib":592.5078125},{"t_s":37,"rss_mib":592.5078125},{"t_s":37,"rss_mib":592.5078125},{"t_s":38,"rss_mib":592.5078125},{"t_s":38,"rss_mib":592.5078125},{"t_s":39,"rss_mib":592.5078125},{"t_s":40,"rss_mib":592.5078125},{"t_s":40,"rss_mib":592.5078125},{"t_s":40,"rss_mib":592.5078125},{"t_s":41,"rss_mib":592.5078125},{"t_s":41,"rss_mib":592.5078125},{"t_s":42,"rss_mib":592.5078125},{"t_s":42,"rss_mib":592.5078125},{"t_s":43,"rss_mib":592.5078125},{"t_s":43,"rss_mib":592.5078125},{"t_s":44,"rss_mib":592.5078125},{"t_s":45,"rss_mib":592.5078125},{"t_s":45,"rss_mib":592.5078125},{"t_s":45,"rss_mib":592.5078125},{"t_s":46,"rss_mib":592.5078125},{"t_s":46,"rss_mib":592.5078125},{"t_s":47,"rss_mib":592.5078125},{"t_s":47,"rss_mib":592.5078125},{"t_s":48,"rss_mib":592.5078125},{"t_s":48,"rss_mib":592.5078125},{"t_s":49,"rss_mib":592.5078125},{"t_s":50,"rss_mib":592.5078125},{"t_s":50,"rss_mib":592.5078125},{"t_s":50,"rss_mib":592.5078125},{"t_s":51,"rss_mib":592.5078125},{"t_s":51,"rss_mib":592.5078125},{"t_s":52,"rss_mib":592.5078125},{"t_s":52,"rss_mib":592.5078125},{"t_s":53,"rss_mib":592.5078125},{"t_s":53,"rss_mib":592.5078125},{"t_s":54,"rss_mib":592.5078125},{"t_s":55,"rss_mib":592.5078125},{"t_s":55,"rss_mib":592.5078125},{"t_s":55,"rss_mib":592.5078125},{"t_s":56,"rss_mib":592.5078125},{"t_s":56,"rss_mib":592.5078125},{"t_s":57,"rss_mib":592.5078125},{"t_s":57,"rss_mib":592.5078125},{"t_s":58,"rss_mib":592.5078125},{"t_s":58,"rss_mib":592.5078125},{"t_s":59,"rss_mib":592.5078125},{"t_s":59,"rss_mib":592.5078125}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.015069088,"cost":6.070009856,"memory":438.746775095,"streaming":0.000002254,"streams_sustained":0.000001481,"throughput":643.029919114},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}}},"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":14005,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":14745,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":14772,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":14262.295081967213,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":70.11494252873564,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.24985541302136363,"certified":true,"suppressed":false},"cost_window_ok":{"value":427,"certified":true,"suppressed":false},"cost_window_rps":{"value":71,"certified":true,"suppressed":false},"cost_threads":{"value":17,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.0468384074941452,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":72,"certified":true,"suppressed":false},"concurrency":2,"p99_us":28461,"first_disqualified_conc":4,"lower_bound":false,"samples":434},{"bound_ms":100,"rps":{"value":72,"certified":true,"suppressed":false},"concurrency":2,"p99_us":28461,"first_disqualified_conc":8,"lower_bound":false,"samples":434},{"bound_ms":null,"rps":{"value":72,"certified":true,"suppressed":false},"concurrency":2,"p99_us":28461,"first_disqualified_conc":8192,"lower_bound":false,"samples":434}],"sweep":[{"conc":1,"ok":431,"rps":71,"p99_us":14456,"fail":0},{"conc":1,"ok":432,"rps":71,"p99_us":14543,"fail":0},{"conc":1,"ok":431,"rps":71,"p99_us":14530,"fail":0},{"conc":2,"ok":434,"rps":72,"p99_us":28402,"fail":0},{"conc":2,"ok":434,"rps":72,"p99_us":28461,"fail":0},{"conc":2,"ok":432,"rps":71,"p99_us":28445,"fail":0},{"conc":4,"ok":430,"rps":71,"p99_us":58185,"fail":0},{"conc":4,"ok":430,"rps":71,"p99_us":58745,"fail":0},{"conc":4,"ok":428,"rps":71,"p99_us":59691,"fail":0},{"conc":8,"ok":429,"rps":71,"p99_us":117964,"fail":0},{"conc":8,"ok":429,"rps":71,"p99_us":116910,"fail":0},{"conc":8,"ok":429,"rps":71,"p99_us":119339,"fail":0},{"conc":16,"ok":429,"rps":71,"p99_us":238615,"fail":0},{"conc":16,"ok":430,"rps":71,"p99_us":238727,"fail":0},{"conc":16,"ok":431,"rps":71,"p99_us":235250,"fail":0},{"conc":32,"ok":427,"rps":71,"p99_us":685797,"fail":0},{"conc":32,"ok":393,"rps":65,"p99_us":942966,"fail":0},{"conc":32,"ok":428,"rps":71,"p99_us":577521,"fail":0},{"conc":64,"ok":420,"rps":69,"p99_us":2263737,"fail":0},{"conc":64,"ok":421,"rps":70,"p99_us":2276242,"fail":0},{"conc":64,"ok":415,"rps":69,"p99_us":2302615,"fail":0},{"conc":128,"ok":386,"rps":64,"p99_us":3394954,"fail":0},{"conc":128,"ok":358,"rps":59,"p99_us":5217879,"fail":0},{"conc":128,"ok":384,"rps":63,"p99_us":1851856,"fail":0},{"conc":256,"ok":279,"rps":46,"p99_us":3775544,"fail":0},{"conc":256,"ok":399,"rps":66,"p99_us":5615532,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":2979593,"fail":0},{"conc":512,"ok":376,"rps":62,"p99_us":5578446,"fail":0},{"conc":512,"ok":321,"rps":53,"p99_us":4554326,"fail":0},{"conc":512,"ok":323,"rps":53,"p99_us":4628175,"fail":0},{"conc":1024,"ok":362,"rps":60,"p99_us":5237297,"fail":0},{"conc":1024,"ok":364,"rps":60,"p99_us":5600901,"fail":0},{"conc":1024,"ok":356,"rps":59,"p99_us":4642484,"fail":0},{"conc":2048,"ok":401,"rps":66,"p99_us":5515226,"fail":0},{"conc":2048,"ok":102,"rps":16,"p99_us":5912052,"fail":0},{"conc":2048,"ok":155,"rps":25,"p99_us":5855279,"fail":0},{"conc":4096,"ok":150,"rps":24,"p99_us":5738838,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":272},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":590}],"rungs":[{"conc":1,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":14530,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":72,"certified":true,"suppressed":false},"p99_us":28445,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":58745,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":117964,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":238615,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":685797,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":69,"certified":true,"suppressed":false},"p99_us":2276242,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":3394954,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":46,"certified":true,"suppressed":false},"p99_us":3775544,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":53,"certified":true,"suppressed":false},"p99_us":4628175,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":5237297,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":25,"certified":true,"suppressed":false},"p99_us":5855279,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":24,"certified":true,"suppressed":false},"p99_us":5738838,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 427 successful gateway round trip(s) and 272464 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":427,"direct_c1_samples":272464},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":593.1875,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":689.1015625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":593.1875},{"t_s":2,"rss_mib":687.9453125},{"t_s":2,"rss_mib":687.97265625},{"t_s":5,"rss_mib":688.17578125},{"t_s":6,"rss_mib":688.1796875},{"t_s":8,"rss_mib":688.37890625},{"t_s":9,"rss_mib":688.38671875},{"t_s":11,"rss_mib":688.53515625},{"t_s":12,"rss_mib":688.5625},{"t_s":14,"rss_mib":688.69921875},{"t_s":15,"rss_mib":688.69921875},{"t_s":17,"rss_mib":688.8359375},{"t_s":18,"rss_mib":688.8359375},{"t_s":20,"rss_mib":688.85546875},{"t_s":21,"rss_mib":688.85546875},{"t_s":23,"rss_mib":688.90234375},{"t_s":24,"rss_mib":688.90234375},{"t_s":27,"rss_mib":688.953125},{"t_s":27,"rss_mib":688.953125},{"t_s":28,"rss_mib":688.9921875},{"t_s":30,"rss_mib":688.99609375},{"t_s":32,"rss_mib":689},{"t_s":33,"rss_mib":689},{"t_s":35,"rss_mib":689.01171875},{"t_s":36,"rss_mib":689.01171875},{"t_s":39,"rss_mib":689.01171875},{"t_s":42,"rss_mib":689.03515625},{"t_s":42,"rss_mib":689.03515625},{"t_s":45,"rss_mib":689.03515625},{"t_s":45,"rss_mib":689.0390625},{"t_s":48,"rss_mib":689.0390625},{"t_s":51,"rss_mib":689.04296875},{"t_s":51,"rss_mib":689.04296875},{"t_s":53,"rss_mib":689.046875},{"t_s":54,"rss_mib":689.046875},{"t_s":57,"rss_mib":689.046875},{"t_s":60,"rss_mib":689.046875},{"t_s":60,"rss_mib":689.05078125},{"t_s":63,"rss_mib":689.05078125},{"t_s":66,"rss_mib":689.05078125},{"t_s":69,"rss_mib":689.05078125},{"t_s":72,"rss_mib":689.05078125},{"t_s":75,"rss_mib":689.05078125},{"t_s":78,"rss_mib":689.0546875},{"t_s":78,"rss_mib":689.0546875},{"t_s":80,"rss_mib":689.06640625},{"t_s":81,"rss_mib":689.06640625},{"t_s":83,"rss_mib":689.0703125},{"t_s":84,"rss_mib":689.0703125},{"t_s":87,"rss_mib":689.0703125},{"t_s":90,"rss_mib":689.0703125},{"t_s":92,"rss_mib":689.078125},{"t_s":93,"rss_mib":689.078125},{"t_s":96,"rss_mib":689.078125},{"t_s":99,"rss_mib":689.078125},{"t_s":102,"rss_mib":689.078125},{"t_s":105,"rss_mib":689.078125},{"t_s":108,"rss_mib":689.078125},{"t_s":111,"rss_mib":689.078125},{"t_s":114,"rss_mib":689.078125},{"t_s":117,"rss_mib":689.078125},{"t_s":120,"rss_mib":689.078125},{"t_s":124,"rss_mib":689.078125},{"t_s":126,"rss_mib":689.078125},{"t_s":130,"rss_mib":689.078125},{"t_s":133,"rss_mib":689.078125},{"t_s":136,"rss_mib":689.078125},{"t_s":139,"rss_mib":689.078125},{"t_s":142,"rss_mib":689.078125},{"t_s":145,"rss_mib":689.078125},{"t_s":148,"rss_mib":689.08203125},{"t_s":151,"rss_mib":689.08203125},{"t_s":154,"rss_mib":689.08203125},{"t_s":157,"rss_mib":689.08203125},{"t_s":160,"rss_mib":689.08203125},{"t_s":163,"rss_mib":689.08203125},{"t_s":166,"rss_mib":689.08203125},{"t_s":168,"rss_mib":689.0859375},{"t_s":169,"rss_mib":689.0859375},{"t_s":172,"rss_mib":689.0859375},{"t_s":175,"rss_mib":689.0859375},{"t_s":178,"rss_mib":689.0859375},{"t_s":181,"rss_mib":689.0859375},{"t_s":184,"rss_mib":689.0859375},{"t_s":187,"rss_mib":689.0859375},{"t_s":190,"rss_mib":689.0859375},{"t_s":193,"rss_mib":689.0859375},{"t_s":196,"rss_mib":689.0859375},{"t_s":199,"rss_mib":689.0859375},{"t_s":202,"rss_mib":689.09375},{"t_s":202,"rss_mib":689.09375},{"t_s":204,"rss_mib":689.1015625},{"t_s":205,"rss_mib":689.1015625},{"t_s":208,"rss_mib":689.1015625},{"t_s":211,"rss_mib":689.1015625},{"t_s":214,"rss_mib":689.1015625},{"t_s":217,"rss_mib":689.1015625},{"t_s":220,"rss_mib":689.1015625},{"t_s":223,"rss_mib":689.1015625},{"t_s":226,"rss_mib":689.1015625},{"t_s":229,"rss_mib":689.1015625},{"t_s":232,"rss_mib":689.1015625},{"t_s":235,"rss_mib":689.1015625},{"t_s":238,"rss_mib":689.1015625},{"t_s":241,"rss_mib":689.1015625},{"t_s":244,"rss_mib":689.1015625},{"t_s":248,"rss_mib":689.1015625},{"t_s":251,"rss_mib":689.1015625},{"t_s":254,"rss_mib":689.1015625},{"t_s":257,"rss_mib":689.1015625},{"t_s":260,"rss_mib":689.1015625},{"t_s":263,"rss_mib":689.1015625},{"t_s":266,"rss_mib":689.1015625},{"t_s":269,"rss_mib":689.1015625},{"t_s":272,"rss_mib":689.1015625},{"t_s":275,"rss_mib":689.1015625},{"t_s":278,"rss_mib":689.1015625},{"t_s":281,"rss_mib":689.1015625},{"t_s":284,"rss_mib":689.1015625},{"t_s":287,"rss_mib":689.1015625},{"t_s":290,"rss_mib":689.1015625},{"t_s":293,"rss_mib":689.1015625},{"t_s":296,"rss_mib":689.1015625},{"t_s":299,"rss_mib":689.1015625},{"t_s":302,"rss_mib":689.1015625},{"t_s":305,"rss_mib":689.1015625},{"t_s":308,"rss_mib":689.1015625},{"t_s":311,"rss_mib":689.1015625},{"t_s":314,"rss_mib":689.1015625},{"t_s":317,"rss_mib":689.1015625},{"t_s":320,"rss_mib":689.1015625},{"t_s":323,"rss_mib":689.1015625},{"t_s":326,"rss_mib":689.1015625},{"t_s":329,"rss_mib":689.1015625},{"t_s":332,"rss_mib":689.1015625},{"t_s":335,"rss_mib":689.1015625},{"t_s":338,"rss_mib":689.1015625},{"t_s":341,"rss_mib":689.1015625},{"t_s":344,"rss_mib":689.1015625},{"t_s":347,"rss_mib":689.1015625},{"t_s":350,"rss_mib":689.1015625},{"t_s":353,"rss_mib":689.1015625},{"t_s":356,"rss_mib":689.1015625},{"t_s":359,"rss_mib":689.1015625},{"t_s":362,"rss_mib":689.1015625}],"idle_rss_series":[{"t_s":0,"rss_mib":593.1875},{"t_s":0,"rss_mib":593.1875},{"t_s":0,"rss_mib":593.1875},{"t_s":1,"rss_mib":593.1875},{"t_s":1,"rss_mib":593.1875},{"t_s":2,"rss_mib":593.1875},{"t_s":2,"rss_mib":593.1875},{"t_s":3,"rss_mib":593.1875},{"t_s":3,"rss_mib":593.1875},{"t_s":4,"rss_mib":593.1875},{"t_s":5,"rss_mib":593.1875},{"t_s":5,"rss_mib":593.1875},{"t_s":5,"rss_mib":593.1875},{"t_s":6,"rss_mib":593.1875},{"t_s":6,"rss_mib":593.1875},{"t_s":7,"rss_mib":593.1875},{"t_s":7,"rss_mib":593.1875},{"t_s":8,"rss_mib":593.1875},{"t_s":8,"rss_mib":593.1875},{"t_s":9,"rss_mib":593.1875},{"t_s":10,"rss_mib":593.1875},{"t_s":10,"rss_mib":593.1875},{"t_s":10,"rss_mib":593.1875},{"t_s":11,"rss_mib":593.1875},{"t_s":11,"rss_mib":593.1875},{"t_s":12,"rss_mib":593.1875},{"t_s":12,"rss_mib":593.1875},{"t_s":13,"rss_mib":593.1875},{"t_s":13,"rss_mib":593.1875},{"t_s":14,"rss_mib":593.1875},{"t_s":15,"rss_mib":593.1875},{"t_s":15,"rss_mib":593.1875},{"t_s":15,"rss_mib":593.1875},{"t_s":16,"rss_mib":593.1875},{"t_s":16,"rss_mib":593.1875},{"t_s":17,"rss_mib":593.1875},{"t_s":17,"rss_mib":593.1875},{"t_s":18,"rss_mib":593.1875},{"t_s":18,"rss_mib":593.1875},{"t_s":19,"rss_mib":593.1875},{"t_s":20,"rss_mib":593.1875},{"t_s":20,"rss_mib":593.1875},{"t_s":20,"rss_mib":593.1875},{"t_s":21,"rss_mib":593.1875},{"t_s":21,"rss_mib":593.1875},{"t_s":22,"rss_mib":593.1875},{"t_s":22,"rss_mib":593.1875},{"t_s":23,"rss_mib":593.1875},{"t_s":23,"rss_mib":593.1875},{"t_s":24,"rss_mib":593.1875},{"t_s":25,"rss_mib":593.1875},{"t_s":25,"rss_mib":593.1875},{"t_s":25,"rss_mib":593.1875},{"t_s":26,"rss_mib":593.1875},{"t_s":26,"rss_mib":593.1875},{"t_s":27,"rss_mib":593.1875},{"t_s":27,"rss_mib":593.1875},{"t_s":28,"rss_mib":593.1875},{"t_s":29,"rss_mib":593.1875},{"t_s":29,"rss_mib":593.1875},{"t_s":30,"rss_mib":593.1875},{"t_s":30,"rss_mib":593.1875},{"t_s":30,"rss_mib":593.1875},{"t_s":31,"rss_mib":593.1875},{"t_s":31,"rss_mib":593.1875},{"t_s":32,"rss_mib":593.1875},{"t_s":32,"rss_mib":593.1875},{"t_s":33,"rss_mib":593.1875},{"t_s":34,"rss_mib":593.1875},{"t_s":34,"rss_mib":593.1875},{"t_s":35,"rss_mib":593.1875},{"t_s":35,"rss_mib":593.1875},{"t_s":35,"rss_mib":593.1875},{"t_s":36,"rss_mib":593.1875},{"t_s":36,"rss_mib":593.1875},{"t_s":37,"rss_mib":593.1875},{"t_s":37,"rss_mib":593.1875},{"t_s":38,"rss_mib":593.1875},{"t_s":39,"rss_mib":593.1875},{"t_s":39,"rss_mib":593.1875},{"t_s":40,"rss_mib":593.1875},{"t_s":40,"rss_mib":593.1875},{"t_s":40,"rss_mib":593.1875},{"t_s":41,"rss_mib":593.1875},{"t_s":41,"rss_mib":593.1875},{"t_s":42,"rss_mib":593.1875},{"t_s":42,"rss_mib":593.1875},{"t_s":43,"rss_mib":593.1875},{"t_s":44,"rss_mib":593.1875},{"t_s":44,"rss_mib":593.1875},{"t_s":45,"rss_mib":593.1875},{"t_s":45,"rss_mib":593.1875},{"t_s":45,"rss_mib":593.1875},{"t_s":46,"rss_mib":593.1875},{"t_s":46,"rss_mib":593.1875},{"t_s":47,"rss_mib":593.1875},{"t_s":47,"rss_mib":593.1875},{"t_s":48,"rss_mib":593.1875},{"t_s":49,"rss_mib":593.1875},{"t_s":49,"rss_mib":593.1875},{"t_s":50,"rss_mib":593.1875},{"t_s":50,"rss_mib":593.1875},{"t_s":50,"rss_mib":593.1875},{"t_s":51,"rss_mib":593.1875},{"t_s":51,"rss_mib":593.1875},{"t_s":52,"rss_mib":593.1875},{"t_s":52,"rss_mib":593.1875},{"t_s":53,"rss_mib":593.1875},{"t_s":54,"rss_mib":593.1875},{"t_s":54,"rss_mib":593.1875},{"t_s":55,"rss_mib":593.1875},{"t_s":55,"rss_mib":593.1875},{"t_s":55,"rss_mib":593.1875},{"t_s":56,"rss_mib":593.1875},{"t_s":56,"rss_mib":593.1875},{"t_s":57,"rss_mib":593.1875},{"t_s":57,"rss_mib":593.1875},{"t_s":58,"rss_mib":593.1875},{"t_s":59,"rss_mib":593.1875},{"t_s":59,"rss_mib":593.1875},{"t_s":59,"rss_mib":593.1875}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.020780002,"cost":6.107125942,"memory":438.416083909,"streaming":0.000002188,"streams_sustained":0.000001955,"throughput":563.062449847},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here"}}}}},"openai":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":8512,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":9522,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":9549,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":8644.793152639088,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":115.67656765676568,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.24990558455671374,"certified":true,"suppressed":false},"cost_window_ok":{"value":701,"certified":true,"suppressed":false},"cost_window_rps":{"value":116,"certified":true,"suppressed":false},"cost_threads":{"value":32,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.028530670470756064,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":117,"certified":true,"suppressed":false},"concurrency":1,"p99_us":9350,"first_disqualified_conc":2,"lower_bound":false,"samples":708},{"bound_ms":50,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":8,"lower_bound":false,"samples":718},{"bound_ms":100,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16,"lower_bound":false,"samples":718},{"bound_ms":null,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16384,"lower_bound":false,"samples":718}],"sweep":[{"conc":1,"ok":708,"rps":117,"p99_us":9178,"fail":0},{"conc":1,"ok":700,"rps":116,"p99_us":9310,"fail":0},{"conc":1,"ok":708,"rps":117,"p99_us":9350,"fail":0},{"conc":2,"ok":721,"rps":119,"p99_us":17781,"fail":0},{"conc":2,"ok":716,"rps":119,"p99_us":17810,"fail":0},{"conc":2,"ok":718,"rps":119,"p99_us":17670,"fail":0},{"conc":4,"ok":712,"rps":118,"p99_us":36003,"fail":0},{"conc":4,"ok":712,"rps":118,"p99_us":35805,"fail":0},{"conc":4,"ok":714,"rps":118,"p99_us":35485,"fail":0},{"conc":8,"ok":712,"rps":118,"p99_us":72900,"fail":0},{"conc":8,"ok":713,"rps":118,"p99_us":72270,"fail":0},{"conc":8,"ok":714,"rps":118,"p99_us":72240,"fail":0},{"conc":16,"ok":716,"rps":119,"p99_us":145952,"fail":0},{"conc":16,"ok":716,"rps":119,"p99_us":146710,"fail":0},{"conc":16,"ok":720,"rps":119,"p99_us":144717,"fail":0},{"conc":32,"ok":720,"rps":119,"p99_us":335625,"fail":0},{"conc":32,"ok":720,"rps":119,"p99_us":314783,"fail":0},{"conc":32,"ok":719,"rps":119,"p99_us":390651,"fail":0},{"conc":64,"ok":721,"rps":119,"p99_us":851275,"fail":0},{"conc":64,"ok":717,"rps":119,"p99_us":1384370,"fail":0},{"conc":64,"ok":719,"rps":119,"p99_us":568535,"fail":0},{"conc":128,"ok":689,"rps":114,"p99_us":5338973,"fail":0},{"conc":128,"ok":642,"rps":106,"p99_us":5507338,"fail":0},{"conc":128,"ok":640,"rps":105,"p99_us":1107255,"fail":0},{"conc":256,"ok":678,"rps":112,"p99_us":5466955,"fail":0},{"conc":256,"ok":630,"rps":104,"p99_us":4921753,"fail":0},{"conc":256,"ok":684,"rps":113,"p99_us":5494141,"fail":0},{"conc":512,"ok":646,"rps":107,"p99_us":4656402,"fail":0},{"conc":512,"ok":666,"rps":110,"p99_us":4568692,"fail":0},{"conc":512,"ok":674,"rps":112,"p99_us":5381288,"fail":0},{"conc":1024,"ok":520,"rps":86,"p99_us":4432245,"fail":0},{"conc":1024,"ok":696,"rps":115,"p99_us":5397311,"fail":0},{"conc":1024,"ok":629,"rps":104,"p99_us":4549641,"fail":0},{"conc":2048,"ok":623,"rps":102,"p99_us":3826765,"fail":0},{"conc":2048,"ok":467,"rps":77,"p99_us":4669951,"fail":0},{"conc":2048,"ok":656,"rps":109,"p99_us":5242341,"fail":0},{"conc":4096,"ok":429,"rps":71,"p99_us":3777750,"fail":0},{"conc":4096,"ok":523,"rps":87,"p99_us":5632784,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":91},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":684},{"conc":8192,"ok":548,"rps":91,"p99_us":5411558,"fail":0},{"conc":8192,"ok":12,"rps":1,"p99_us":5960300,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":2376},{"conc":16384,"ok":33,"rps":5,"p99_us":5965146,"fail":1028},{"conc":16384,"ok":551,"rps":91,"p99_us":4408916,"fail":4095}],"rungs":[{"conc":1,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":9310,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":17781,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":35805,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":72270,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":145952,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":335625,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":851275,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":106,"certified":true,"suppressed":false},"p99_us":5338973,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":112,"certified":true,"suppressed":false},"p99_us":5466955,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":110,"certified":true,"suppressed":false},"p99_us":4656402,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":104,"certified":true,"suppressed":false},"p99_us":4549641,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":4669951,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":3777750,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":1,"certified":true,"suppressed":false},"p99_us":5411558,"windows":3,"clean_windows":2},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"openai>openai is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 701 successful gateway round trip(s) and 267297 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":701,"direct_c1_samples":267297},"stream":{"stream_served":true,"stream_error":null,"reason":null,"stream_c1_note":"the c=1 streaming legs read 64 frame(s) through the gateway and 64 direct to the mock, out of a 64 frame budget. The added-GAP figures are percentiles over the intervals inside those two streams. The added-TTFT figures are percentiles over separate probes - 100 through the gateway and 100 direct, out of 100 attempted per leg - so they carry that weight and no more","added_ttft_p50_us":{"value":11154,"certified":true,"suppressed":false},"added_ttft_p99_us":{"value":11974,"certified":true,"suppressed":false},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21070us) came in under the mock's (21072us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing"},"added_gap_p99_us":{"value":9,"certified":true,"suppressed":false},"streams_sustained_fps":{"value":1207.9948436569616,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494},"streams_sustained":{"value":62,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494},"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.95703125,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":628.60546875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.95703125},{"t_s":2,"rss_mib":628.1796875},{"t_s":2,"rss_mib":628.1796875},{"t_s":5,"rss_mib":628.375},{"t_s":6,"rss_mib":628.375},{"t_s":8,"rss_mib":628.4921875},{"t_s":8,"rss_mib":628.5},{"t_s":11,"rss_mib":628.6015625},{"t_s":12,"rss_mib":628.6015625},{"t_s":12,"rss_mib":628.60546875},{"t_s":15,"rss_mib":628.60546875},{"t_s":18,"rss_mib":628.60546875},{"t_s":21,"rss_mib":628.60546875},{"t_s":24,"rss_mib":628.60546875},{"t_s":27,"rss_mib":628.60546875},{"t_s":30,"rss_mib":628.60546875},{"t_s":33,"rss_mib":628.60546875},{"t_s":36,"rss_mib":628.60546875},{"t_s":39,"rss_mib":628.60546875},{"t_s":42,"rss_mib":628.60546875},{"t_s":45,"rss_mib":628.60546875},{"t_s":48,"rss_mib":628.60546875},{"t_s":51,"rss_mib":628.60546875},{"t_s":54,"rss_mib":628.60546875},{"t_s":57,"rss_mib":628.60546875},{"t_s":60,"rss_mib":628.60546875},{"t_s":63,"rss_mib":628.60546875},{"t_s":66,"rss_mib":628.60546875},{"t_s":69,"rss_mib":628.60546875},{"t_s":72,"rss_mib":628.60546875},{"t_s":75,"rss_mib":628.60546875},{"t_s":78,"rss_mib":628.60546875},{"t_s":81,"rss_mib":628.60546875},{"t_s":84,"rss_mib":628.60546875},{"t_s":87,"rss_mib":628.60546875},{"t_s":90,"rss_mib":628.60546875},{"t_s":93,"rss_mib":628.60546875},{"t_s":96,"rss_mib":628.60546875},{"t_s":99,"rss_mib":628.60546875},{"t_s":102,"rss_mib":628.60546875},{"t_s":105,"rss_mib":628.60546875},{"t_s":108,"rss_mib":628.60546875},{"t_s":111,"rss_mib":628.60546875},{"t_s":114,"rss_mib":628.60546875},{"t_s":117,"rss_mib":628.60546875},{"t_s":120,"rss_mib":628.60546875},{"t_s":123,"rss_mib":628.60546875},{"t_s":126,"rss_mib":628.60546875},{"t_s":129,"rss_mib":628.60546875},{"t_s":132,"rss_mib":628.60546875},{"t_s":135,"rss_mib":628.60546875},{"t_s":138,"rss_mib":628.60546875},{"t_s":141,"rss_mib":628.60546875},{"t_s":144,"rss_mib":628.60546875},{"t_s":147,"rss_mib":628.60546875},{"t_s":151,"rss_mib":628.60546875},{"t_s":153,"rss_mib":628.60546875},{"t_s":157,"rss_mib":628.60546875},{"t_s":160,"rss_mib":628.60546875},{"t_s":163,"rss_mib":628.60546875},{"t_s":166,"rss_mib":628.60546875},{"t_s":169,"rss_mib":628.60546875},{"t_s":172,"rss_mib":628.60546875},{"t_s":175,"rss_mib":628.60546875},{"t_s":178,"rss_mib":628.60546875},{"t_s":181,"rss_mib":628.60546875},{"t_s":184,"rss_mib":628.60546875},{"t_s":187,"rss_mib":628.60546875},{"t_s":190,"rss_mib":628.60546875},{"t_s":193,"rss_mib":628.60546875},{"t_s":196,"rss_mib":628.60546875},{"t_s":199,"rss_mib":628.60546875},{"t_s":202,"rss_mib":628.60546875},{"t_s":205,"rss_mib":628.60546875},{"t_s":208,"rss_mib":628.60546875},{"t_s":211,"rss_mib":628.60546875},{"t_s":214,"rss_mib":628.60546875},{"t_s":217,"rss_mib":628.60546875},{"t_s":220,"rss_mib":628.60546875},{"t_s":223,"rss_mib":628.60546875},{"t_s":226,"rss_mib":628.60546875},{"t_s":229,"rss_mib":628.60546875},{"t_s":232,"rss_mib":628.60546875},{"t_s":235,"rss_mib":628.60546875},{"t_s":238,"rss_mib":628.60546875},{"t_s":241,"rss_mib":628.60546875},{"t_s":244,"rss_mib":628.60546875},{"t_s":247,"rss_mib":628.60546875},{"t_s":250,"rss_mib":628.60546875},{"t_s":253,"rss_mib":628.60546875},{"t_s":256,"rss_mib":628.60546875},{"t_s":259,"rss_mib":628.60546875},{"t_s":262,"rss_mib":628.60546875},{"t_s":265,"rss_mib":628.60546875},{"t_s":268,"rss_mib":628.60546875},{"t_s":271,"rss_mib":628.60546875},{"t_s":274,"rss_mib":628.60546875},{"t_s":277,"rss_mib":628.60546875},{"t_s":280,"rss_mib":628.60546875},{"t_s":283,"rss_mib":628.60546875},{"t_s":286,"rss_mib":628.60546875},{"t_s":289,"rss_mib":628.60546875},{"t_s":292,"rss_mib":628.60546875},{"t_s":295,"rss_mib":628.60546875},{"t_s":299,"rss_mib":628.60546875},{"t_s":301,"rss_mib":628.60546875},{"t_s":305,"rss_mib":628.60546875},{"t_s":308,"rss_mib":628.60546875},{"t_s":311,"rss_mib":628.60546875},{"t_s":314,"rss_mib":628.60546875},{"t_s":317,"rss_mib":628.60546875},{"t_s":320,"rss_mib":628.60546875},{"t_s":323,"rss_mib":628.60546875},{"t_s":326,"rss_mib":628.60546875},{"t_s":329,"rss_mib":628.60546875},{"t_s":332,"rss_mib":628.60546875},{"t_s":335,"rss_mib":628.60546875},{"t_s":338,"rss_mib":628.60546875},{"t_s":341,"rss_mib":628.60546875},{"t_s":344,"rss_mib":628.60546875},{"t_s":347,"rss_mib":628.60546875},{"t_s":350,"rss_mib":628.60546875},{"t_s":353,"rss_mib":628.60546875},{"t_s":356,"rss_mib":628.60546875},{"t_s":359,"rss_mib":628.60546875},{"t_s":362,"rss_mib":628.60546875}],"idle_rss_series":[{"t_s":0,"rss_mib":592.95703125},{"t_s":0,"rss_mib":592.95703125},{"t_s":0,"rss_mib":592.95703125},{"t_s":1,"rss_mib":592.95703125},{"t_s":1,"rss_mib":592.95703125},{"t_s":2,"rss_mib":592.95703125},{"t_s":2,"rss_mib":592.95703125},{"t_s":3,"rss_mib":592.95703125},{"t_s":3,"rss_mib":592.95703125},{"t_s":4,"rss_mib":592.95703125},{"t_s":5,"rss_mib":592.95703125},{"t_s":5,"rss_mib":592.95703125},{"t_s":5,"rss_mib":592.95703125},{"t_s":6,"rss_mib":592.95703125},{"t_s":6,"rss_mib":592.95703125},{"t_s":7,"rss_mib":592.95703125},{"t_s":7,"rss_mib":592.95703125},{"t_s":8,"rss_mib":592.95703125},{"t_s":8,"rss_mib":592.95703125},{"t_s":9,"rss_mib":592.95703125},{"t_s":10,"rss_mib":592.95703125},{"t_s":10,"rss_mib":592.95703125},{"t_s":10,"rss_mib":592.95703125},{"t_s":11,"rss_mib":592.95703125},{"t_s":11,"rss_mib":592.95703125},{"t_s":12,"rss_mib":592.95703125},{"t_s":12,"rss_mib":592.95703125},{"t_s":13,"rss_mib":592.95703125},{"t_s":13,"rss_mib":592.95703125},{"t_s":14,"rss_mib":592.95703125},{"t_s":15,"rss_mib":592.95703125},{"t_s":15,"rss_mib":592.95703125},{"t_s":16,"rss_mib":592.95703125},{"t_s":16,"rss_mib":592.95703125},{"t_s":16,"rss_mib":592.95703125},{"t_s":17,"rss_mib":592.95703125},{"t_s":17,"rss_mib":592.95703125},{"t_s":18,"rss_mib":592.95703125},{"t_s":18,"rss_mib":592.95703125},{"t_s":19,"rss_mib":592.95703125},{"t_s":20,"rss_mib":592.95703125},{"t_s":20,"rss_mib":592.95703125},{"t_s":21,"rss_mib":592.95703125},{"t_s":21,"rss_mib":592.95703125},{"t_s":21,"rss_mib":592.95703125},{"t_s":22,"rss_mib":592.95703125},{"t_s":22,"rss_mib":592.95703125},{"t_s":23,"rss_mib":592.95703125},{"t_s":23,"rss_mib":592.95703125},{"t_s":24,"rss_mib":592.95703125},{"t_s":25,"rss_mib":592.95703125},{"t_s":25,"rss_mib":592.95703125},{"t_s":26,"rss_mib":592.95703125},{"t_s":26,"rss_mib":592.95703125},{"t_s":27,"rss_mib":592.95703125},{"t_s":27,"rss_mib":592.95703125},{"t_s":27,"rss_mib":592.95703125},{"t_s":28,"rss_mib":592.95703125},{"t_s":28,"rss_mib":592.95703125},{"t_s":29,"rss_mib":592.95703125},{"t_s":30,"rss_mib":592.95703125},{"t_s":30,"rss_mib":592.95703125},{"t_s":31,"rss_mib":592.95703125},{"t_s":31,"rss_mib":592.95703125},{"t_s":32,"rss_mib":592.95703125},{"t_s":32,"rss_mib":592.95703125},{"t_s":32,"rss_mib":592.95703125},{"t_s":33,"rss_mib":592.95703125},{"t_s":33,"rss_mib":592.95703125},{"t_s":34,"rss_mib":592.95703125},{"t_s":35,"rss_mib":592.95703125},{"t_s":35,"rss_mib":592.95703125},{"t_s":36,"rss_mib":592.95703125},{"t_s":36,"rss_mib":592.95703125},{"t_s":37,"rss_mib":592.95703125},{"t_s":37,"rss_mib":592.95703125},{"t_s":38,"rss_mib":592.95703125},{"t_s":38,"rss_mib":592.95703125},{"t_s":38,"rss_mib":592.95703125},{"t_s":39,"rss_mib":592.95703125},{"t_s":40,"rss_mib":592.95703125},{"t_s":40,"rss_mib":592.95703125},{"t_s":41,"rss_mib":592.95703125},{"t_s":41,"rss_mib":592.95703125},{"t_s":42,"rss_mib":592.95703125},{"t_s":42,"rss_mib":592.95703125},{"t_s":43,"rss_mib":592.95703125},{"t_s":43,"rss_mib":592.95703125},{"t_s":43,"rss_mib":592.95703125},{"t_s":44,"rss_mib":592.95703125},{"t_s":45,"rss_mib":592.95703125},{"t_s":45,"rss_mib":592.95703125},{"t_s":46,"rss_mib":592.95703125},{"t_s":46,"rss_mib":592.95703125},{"t_s":47,"rss_mib":592.95703125},{"t_s":47,"rss_mib":592.95703125},{"t_s":48,"rss_mib":592.95703125},{"t_s":48,"rss_mib":592.95703125},{"t_s":49,"rss_mib":592.95703125},{"t_s":49,"rss_mib":592.95703125},{"t_s":49,"rss_mib":592.95703125},{"t_s":50,"rss_mib":592.95703125},{"t_s":51,"rss_mib":592.95703125},{"t_s":51,"rss_mib":592.95703125},{"t_s":52,"rss_mib":592.95703125},{"t_s":52,"rss_mib":592.95703125},{"t_s":53,"rss_mib":592.95703125},{"t_s":53,"rss_mib":592.95703125},{"t_s":54,"rss_mib":592.95703125},{"t_s":54,"rss_mib":592.95703125},{"t_s":54,"rss_mib":592.95703125},{"t_s":55,"rss_mib":592.95703125},{"t_s":56,"rss_mib":592.95703125},{"t_s":56,"rss_mib":592.95703125},{"t_s":57,"rss_mib":592.95703125},{"t_s":57,"rss_mib":592.95703125},{"t_s":58,"rss_mib":592.95703125},{"t_s":58,"rss_mib":592.95703125},{"t_s":59,"rss_mib":592.95703125},{"t_s":59,"rss_mib":592.95703125},{"t_s":59,"rss_mib":592.95703125}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.012934796,"cost":6.0757588,"memory":438.157022432,"streaming":3.77281335,"streams_sustained":34.016846677,"throughput":625.286644037},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21070us) came in under the mock's (21072us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing"}}},"anthropic":{"served":"failed","reason":"failed","status":"200","path":"/v1/messages","verdict_note":"probed and answered HTTP 200, but the body is not a usable anthropic response (see body_snippet)","body_snippet":"the 2xx body is not a response of the anthropic dialect with assistant text at content[type=text].text: {\"id\":\"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo2MTk3ZDQ1OWNlMzc1OTE2NzVlMTIyMTg3ZDAwOTViNzIyNDhmMTRmYTUwNzQzMDZkMzFlMzlhODkzMTM2OWQ4O3Jlc3BvbnNlX2lkOmNoYXRjbXBsLXg=\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"gpt-4o-mini\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"output_tokens\":2},\"content\":[],\"stop_reason\":\"end_turn\",\"stop_details\":null}","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10676,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":11606,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":11634,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10555.555555555555,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":94.73684210526315,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2503730077247171,"certified":true,"suppressed":false},"cost_window_ok":{"value":576,"certified":true,"suppressed":false},"cost_window_rps":{"value":95,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.059027777777777776,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":95,"certified":true,"suppressed":false},"concurrency":2,"p99_us":22974,"first_disqualified_conc":8,"lower_bound":false,"samples":572},{"bound_ms":100,"rps":{"value":95,"certified":true,"suppressed":false},"concurrency":2,"p99_us":22974,"first_disqualified_conc":16,"lower_bound":false,"samples":572},{"bound_ms":null,"rps":{"value":95,"certified":true,"suppressed":false},"concurrency":2,"p99_us":22974,"first_disqualified_conc":4096,"lower_bound":false,"samples":572}],"sweep":[{"conc":1,"ok":560,"rps":93,"p99_us":11674,"fail":0},{"conc":1,"ok":561,"rps":93,"p99_us":11695,"fail":0},{"conc":1,"ok":561,"rps":93,"p99_us":11809,"fail":0},{"conc":2,"ok":572,"rps":95,"p99_us":22974,"fail":0},{"conc":2,"ok":568,"rps":94,"p99_us":22515,"fail":0},{"conc":2,"ok":568,"rps":94,"p99_us":22632,"fail":0},{"conc":4,"ok":565,"rps":94,"p99_us":46984,"fail":0},{"conc":4,"ok":565,"rps":94,"p99_us":47157,"fail":0},{"conc":4,"ok":564,"rps":93,"p99_us":47188,"fail":0},{"conc":8,"ok":561,"rps":93,"p99_us":91278,"fail":0},{"conc":8,"ok":556,"rps":92,"p99_us":90773,"fail":0},{"conc":8,"ok":560,"rps":93,"p99_us":91947,"fail":0},{"conc":16,"ok":561,"rps":93,"p99_us":182064,"fail":0},{"conc":16,"ok":563,"rps":93,"p99_us":184316,"fail":0},{"conc":16,"ok":561,"rps":93,"p99_us":182613,"fail":0},{"conc":32,"ok":563,"rps":93,"p99_us":481743,"fail":0},{"conc":32,"ok":558,"rps":92,"p99_us":505194,"fail":0},{"conc":32,"ok":555,"rps":92,"p99_us":497540,"fail":0},{"conc":64,"ok":512,"rps":85,"p99_us":1317740,"fail":0},{"conc":64,"ok":553,"rps":92,"p99_us":1637546,"fail":0},{"conc":64,"ok":576,"rps":95,"p99_us":726447,"fail":0},{"conc":128,"ok":512,"rps":85,"p99_us":1784461,"fail":0},{"conc":128,"ok":512,"rps":85,"p99_us":1789978,"fail":0},{"conc":128,"ok":540,"rps":89,"p99_us":5566564,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":3087975,"fail":0},{"conc":256,"ok":480,"rps":79,"p99_us":5618925,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":3320131,"fail":0},{"conc":512,"ok":521,"rps":86,"p99_us":5410560,"fail":0},{"conc":512,"ok":332,"rps":55,"p99_us":4742425,"fail":0},{"conc":512,"ok":326,"rps":54,"p99_us":4907521,"fail":0},{"conc":1024,"ok":358,"rps":59,"p99_us":4534921,"fail":0},{"conc":1024,"ok":348,"rps":57,"p99_us":5249194,"fail":0},{"conc":1024,"ok":366,"rps":60,"p99_us":5358578,"fail":0},{"conc":2048,"ok":375,"rps":61,"p99_us":5282055,"fail":0},{"conc":2048,"ok":534,"rps":88,"p99_us":5592968,"fail":0},{"conc":2048,"ok":391,"rps":65,"p99_us":4912332,"fail":0},{"conc":4096,"ok":397,"rps":66,"p99_us":5040629,"fail":654},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":93,"certified":true,"suppressed":false},"p99_us":11695,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":94,"certified":true,"suppressed":false},"p99_us":22632,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":94,"certified":true,"suppressed":false},"p99_us":47157,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":93,"certified":true,"suppressed":false},"p99_us":91278,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":93,"certified":true,"suppressed":false},"p99_us":182613,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":497540,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":1317740,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":1789978,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":3320131,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":4907521,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":5249194,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":65,"certified":true,"suppressed":false},"p99_us":5282055,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 1 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 559 successful gateway round trip(s) and 265389 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":559,"direct_c1_samples":265389},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.90625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":629.95703125,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":629.96875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":629.96875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":629.96875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.00555168581824748,"certified":true,"suppressed":false},"load_s":{"value":304,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.90625},{"t_s":2,"rss_mib":628.984375},{"t_s":2,"rss_mib":628.984375},{"t_s":5,"rss_mib":629.203125},{"t_s":6,"rss_mib":629.2109375},{"t_s":8,"rss_mib":629.55078125},{"t_s":9,"rss_mib":629.55078125},{"t_s":11,"rss_mib":629.703125},{"t_s":12,"rss_mib":629.703125},{"t_s":14,"rss_mib":629.75},{"t_s":15,"rss_mib":629.75},{"t_s":18,"rss_mib":629.76171875},{"t_s":18,"rss_mib":629.8125},{"t_s":21,"rss_mib":629.85546875},{"t_s":21,"rss_mib":629.85546875},{"t_s":24,"rss_mib":629.8671875},{"t_s":24,"rss_mib":629.8671875},{"t_s":26,"rss_mib":629.87109375},{"t_s":27,"rss_mib":629.87109375},{"t_s":30,"rss_mib":629.87109375},{"t_s":33,"rss_mib":629.875},{"t_s":36,"rss_mib":629.88671875},{"t_s":36,"rss_mib":629.88671875},{"t_s":39,"rss_mib":629.88671875},{"t_s":42,"rss_mib":629.88671875},{"t_s":44,"rss_mib":629.91796875},{"t_s":45,"rss_mib":629.921875},{"t_s":48,"rss_mib":629.921875},{"t_s":51,"rss_mib":629.921875},{"t_s":52,"rss_mib":629.92578125},{"t_s":54,"rss_mib":629.92578125},{"t_s":57,"rss_mib":629.92578125},{"t_s":60,"rss_mib":629.92578125},{"t_s":61,"rss_mib":629.9296875},{"t_s":63,"rss_mib":629.9296875},{"t_s":66,"rss_mib":629.94140625},{"t_s":66,"rss_mib":629.94140625},{"t_s":68,"rss_mib":629.9453125},{"t_s":69,"rss_mib":629.9453125},{"t_s":70,"rss_mib":629.94921875},{"t_s":72,"rss_mib":629.94921875},{"t_s":75,"rss_mib":629.94921875},{"t_s":78,"rss_mib":629.94921875},{"t_s":82,"rss_mib":629.94921875},{"t_s":85,"rss_mib":629.94921875},{"t_s":88,"rss_mib":629.94921875},{"t_s":91,"rss_mib":629.94921875},{"t_s":94,"rss_mib":629.94921875},{"t_s":97,"rss_mib":629.94921875},{"t_s":100,"rss_mib":629.94921875},{"t_s":103,"rss_mib":629.94921875},{"t_s":106,"rss_mib":629.94921875},{"t_s":109,"rss_mib":629.94921875},{"t_s":112,"rss_mib":629.94921875},{"t_s":115,"rss_mib":629.94921875},{"t_s":118,"rss_mib":629.94921875},{"t_s":121,"rss_mib":629.94921875},{"t_s":124,"rss_mib":629.94921875},{"t_s":127,"rss_mib":629.94921875},{"t_s":130,"rss_mib":629.94921875},{"t_s":133,"rss_mib":629.94921875},{"t_s":136,"rss_mib":629.94921875},{"t_s":139,"rss_mib":629.94921875},{"t_s":142,"rss_mib":629.94921875},{"t_s":145,"rss_mib":629.94921875},{"t_s":148,"rss_mib":629.94921875},{"t_s":151,"rss_mib":629.94921875},{"t_s":155,"rss_mib":629.94921875},{"t_s":157,"rss_mib":629.94921875},{"t_s":161,"rss_mib":629.94921875},{"t_s":161,"rss_mib":629.953125},{"t_s":164,"rss_mib":629.953125},{"t_s":167,"rss_mib":629.953125},{"t_s":170,"rss_mib":629.95703125},{"t_s":173,"rss_mib":629.95703125},{"t_s":176,"rss_mib":629.95703125},{"t_s":179,"rss_mib":629.95703125},{"t_s":182,"rss_mib":629.95703125},{"t_s":185,"rss_mib":629.95703125},{"t_s":188,"rss_mib":629.95703125},{"t_s":191,"rss_mib":629.95703125},{"t_s":194,"rss_mib":629.95703125},{"t_s":197,"rss_mib":629.95703125},{"t_s":200,"rss_mib":629.95703125},{"t_s":203,"rss_mib":629.95703125},{"t_s":206,"rss_mib":629.95703125},{"t_s":209,"rss_mib":629.95703125},{"t_s":212,"rss_mib":629.95703125},{"t_s":215,"rss_mib":629.95703125},{"t_s":218,"rss_mib":629.95703125},{"t_s":221,"rss_mib":629.95703125},{"t_s":224,"rss_mib":629.95703125},{"t_s":227,"rss_mib":629.95703125},{"t_s":231,"rss_mib":629.95703125},{"t_s":234,"rss_mib":629.95703125},{"t_s":237,"rss_mib":629.95703125},{"t_s":240,"rss_mib":629.95703125},{"t_s":243,"rss_mib":629.95703125},{"t_s":243,"rss_mib":629.96484375},{"t_s":246,"rss_mib":629.96484375},{"t_s":249,"rss_mib":629.96484375},{"t_s":252,"rss_mib":629.96484375},{"t_s":255,"rss_mib":629.96484375},{"t_s":258,"rss_mib":629.96484375},{"t_s":261,"rss_mib":629.96484375},{"t_s":264,"rss_mib":629.96484375},{"t_s":267,"rss_mib":629.96484375},{"t_s":268,"rss_mib":629.96875},{"t_s":270,"rss_mib":629.96875},{"t_s":273,"rss_mib":629.96875},{"t_s":276,"rss_mib":629.96875},{"t_s":279,"rss_mib":629.96875},{"t_s":282,"rss_mib":629.96875},{"t_s":285,"rss_mib":629.96875},{"t_s":288,"rss_mib":629.96875},{"t_s":291,"rss_mib":629.96875},{"t_s":294,"rss_mib":629.96875},{"t_s":297,"rss_mib":629.96875},{"t_s":300,"rss_mib":629.96875},{"t_s":303,"rss_mib":629.96875},{"t_s":306,"rss_mib":629.96875},{"t_s":310,"rss_mib":629.96875},{"t_s":313,"rss_mib":629.96875},{"t_s":316,"rss_mib":629.96875},{"t_s":319,"rss_mib":629.96875},{"t_s":322,"rss_mib":629.96875},{"t_s":325,"rss_mib":629.96875},{"t_s":328,"rss_mib":629.96875},{"t_s":331,"rss_mib":629.96875},{"t_s":334,"rss_mib":629.96875},{"t_s":337,"rss_mib":629.96875},{"t_s":340,"rss_mib":629.96875},{"t_s":343,"rss_mib":629.96875},{"t_s":346,"rss_mib":629.96875},{"t_s":349,"rss_mib":629.96875},{"t_s":352,"rss_mib":629.96875},{"t_s":355,"rss_mib":629.96875},{"t_s":358,"rss_mib":629.96875},{"t_s":361,"rss_mib":629.96875},{"t_s":364,"rss_mib":629.96875}],"idle_rss_series":[{"t_s":0,"rss_mib":591.90625},{"t_s":0,"rss_mib":591.90625},{"t_s":0,"rss_mib":591.90625},{"t_s":1,"rss_mib":591.90625},{"t_s":1,"rss_mib":591.90625},{"t_s":2,"rss_mib":591.90625},{"t_s":2,"rss_mib":591.90625},{"t_s":3,"rss_mib":591.90625},{"t_s":4,"rss_mib":591.90625},{"t_s":4,"rss_mib":591.90625},{"t_s":4,"rss_mib":591.90625},{"t_s":5,"rss_mib":591.90625},{"t_s":5,"rss_mib":591.90625},{"t_s":6,"rss_mib":591.90625},{"t_s":6,"rss_mib":591.90625},{"t_s":7,"rss_mib":591.90625},{"t_s":7,"rss_mib":591.90625},{"t_s":8,"rss_mib":591.90625},{"t_s":8,"rss_mib":591.90625},{"t_s":9,"rss_mib":591.90625},{"t_s":9,"rss_mib":591.90625},{"t_s":10,"rss_mib":591.90625},{"t_s":10,"rss_mib":591.90625},{"t_s":11,"rss_mib":591.90625},{"t_s":11,"rss_mib":591.90625},{"t_s":12,"rss_mib":591.90625},{"t_s":12,"rss_mib":591.90625},{"t_s":13,"rss_mib":591.90625},{"t_s":13,"rss_mib":591.90625},{"t_s":14,"rss_mib":591.90625},{"t_s":14,"rss_mib":591.90625},{"t_s":15,"rss_mib":591.90625},{"t_s":15,"rss_mib":591.90625},{"t_s":16,"rss_mib":591.90625},{"t_s":17,"rss_mib":591.90625},{"t_s":17,"rss_mib":591.90625},{"t_s":17,"rss_mib":591.90625},{"t_s":18,"rss_mib":591.90625},{"t_s":18,"rss_mib":591.90625},{"t_s":19,"rss_mib":591.90625},{"t_s":19,"rss_mib":591.90625},{"t_s":20,"rss_mib":591.90625},{"t_s":20,"rss_mib":591.90625},{"t_s":21,"rss_mib":591.90625},{"t_s":21,"rss_mib":591.90625},{"t_s":22,"rss_mib":591.90625},{"t_s":22,"rss_mib":591.90625},{"t_s":23,"rss_mib":591.90625},{"t_s":23,"rss_mib":591.90625},{"t_s":24,"rss_mib":591.90625},{"t_s":24,"rss_mib":591.90625},{"t_s":25,"rss_mib":591.90625},{"t_s":25,"rss_mib":591.90625},{"t_s":26,"rss_mib":591.90625},{"t_s":26,"rss_mib":591.90625},{"t_s":27,"rss_mib":591.90625},{"t_s":27,"rss_mib":591.90625},{"t_s":28,"rss_mib":591.90625},{"t_s":28,"rss_mib":591.90625},{"t_s":29,"rss_mib":591.90625},{"t_s":30,"rss_mib":591.90625},{"t_s":30,"rss_mib":591.90625},{"t_s":30,"rss_mib":591.90625},{"t_s":31,"rss_mib":591.90625},{"t_s":31,"rss_mib":591.90625},{"t_s":32,"rss_mib":591.90625},{"t_s":32,"rss_mib":591.90625},{"t_s":33,"rss_mib":591.90625},{"t_s":34,"rss_mib":591.90625},{"t_s":34,"rss_mib":591.90625},{"t_s":34,"rss_mib":591.90625},{"t_s":35,"rss_mib":591.90625},{"t_s":35,"rss_mib":591.90625},{"t_s":36,"rss_mib":591.90625},{"t_s":36,"rss_mib":591.90625},{"t_s":37,"rss_mib":591.90625},{"t_s":37,"rss_mib":591.90625},{"t_s":38,"rss_mib":591.90625},{"t_s":38,"rss_mib":591.90625},{"t_s":39,"rss_mib":591.90625},{"t_s":39,"rss_mib":591.90625},{"t_s":40,"rss_mib":591.90625},{"t_s":40,"rss_mib":591.90625},{"t_s":41,"rss_mib":591.90625},{"t_s":41,"rss_mib":591.90625},{"t_s":42,"rss_mib":591.90625},{"t_s":42,"rss_mib":591.90625},{"t_s":43,"rss_mib":591.90625},{"t_s":43,"rss_mib":591.90625},{"t_s":44,"rss_mib":591.90625},{"t_s":44,"rss_mib":591.90625},{"t_s":45,"rss_mib":591.90625},{"t_s":45,"rss_mib":591.90625},{"t_s":46,"rss_mib":591.90625},{"t_s":47,"rss_mib":591.90625},{"t_s":47,"rss_mib":591.90625},{"t_s":47,"rss_mib":591.90625},{"t_s":48,"rss_mib":591.90625},{"t_s":48,"rss_mib":591.90625},{"t_s":49,"rss_mib":591.90625},{"t_s":49,"rss_mib":591.90625},{"t_s":50,"rss_mib":591.90625},{"t_s":51,"rss_mib":591.90625},{"t_s":51,"rss_mib":591.90625},{"t_s":51,"rss_mib":591.90625},{"t_s":52,"rss_mib":591.90625},{"t_s":52,"rss_mib":591.90625},{"t_s":53,"rss_mib":591.90625},{"t_s":53,"rss_mib":591.90625},{"t_s":54,"rss_mib":591.90625},{"t_s":54,"rss_mib":591.90625},{"t_s":55,"rss_mib":591.90625},{"t_s":55,"rss_mib":591.90625},{"t_s":56,"rss_mib":591.90625},{"t_s":56,"rss_mib":591.90625},{"t_s":57,"rss_mib":591.90625},{"t_s":57,"rss_mib":591.90625},{"t_s":58,"rss_mib":591.90625},{"t_s":58,"rss_mib":591.90625},{"t_s":59,"rss_mib":591.90625},{"t_s":59,"rss_mib":591.90625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.012968903,"cost":6.084342769,"memory":440.383300128,"streaming":0.000005217,"streams_sustained":0.000001782,"throughput":429.010812803},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"cohere":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/v2/chat","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"bedrock":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/model/gpt-4o-mini/converse","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/gpt-4o-mini:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":14407,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":15293,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":15321,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":15086.41975308642,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":66.28477905073649,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25038840963155784,"certified":true,"suppressed":false},"cost_window_ok":{"value":405,"certified":true,"suppressed":false},"cost_window_rps":{"value":67,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.044444444444444446,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":68,"certified":true,"suppressed":false},"concurrency":1,"p99_us":15465,"first_disqualified_conc":4,"lower_bound":false,"samples":410},{"bound_ms":100,"rps":{"value":68,"certified":true,"suppressed":false},"concurrency":1,"p99_us":15465,"first_disqualified_conc":8,"lower_bound":false,"samples":410},{"bound_ms":null,"rps":{"value":68,"certified":true,"suppressed":false},"concurrency":1,"p99_us":15465,"first_disqualified_conc":8192,"lower_bound":false,"samples":410}],"sweep":[{"conc":1,"ok":410,"rps":68,"p99_us":15406,"fail":0},{"conc":1,"ok":410,"rps":68,"p99_us":15465,"fail":0},{"conc":1,"ok":405,"rps":67,"p99_us":15565,"fail":0},{"conc":2,"ok":408,"rps":67,"p99_us":30727,"fail":0},{"conc":2,"ok":412,"rps":68,"p99_us":29756,"fail":0},{"conc":2,"ok":412,"rps":68,"p99_us":30232,"fail":0},{"conc":4,"ok":409,"rps":68,"p99_us":62750,"fail":0},{"conc":4,"ok":409,"rps":68,"p99_us":61138,"fail":0},{"conc":4,"ok":412,"rps":68,"p99_us":61409,"fail":0},{"conc":8,"ok":380,"rps":63,"p99_us":542507,"fail":0},{"conc":8,"ok":409,"rps":68,"p99_us":124533,"fail":0},{"conc":8,"ok":411,"rps":68,"p99_us":124403,"fail":0},{"conc":16,"ok":407,"rps":67,"p99_us":248972,"fail":0},{"conc":16,"ok":408,"rps":67,"p99_us":247703,"fail":0},{"conc":16,"ok":409,"rps":68,"p99_us":250322,"fail":0},{"conc":32,"ok":404,"rps":67,"p99_us":707042,"fail":0},{"conc":32,"ok":402,"rps":66,"p99_us":711840,"fail":0},{"conc":32,"ok":404,"rps":67,"p99_us":744787,"fail":0},{"conc":64,"ok":397,"rps":66,"p99_us":2379037,"fail":0},{"conc":64,"ok":363,"rps":60,"p99_us":2435929,"fail":0},{"conc":64,"ok":395,"rps":65,"p99_us":2453410,"fail":0},{"conc":128,"ok":382,"rps":63,"p99_us":5670316,"fail":0},{"conc":128,"ok":384,"rps":63,"p99_us":2395769,"fail":0},{"conc":128,"ok":365,"rps":60,"p99_us":5520133,"fail":0},{"conc":256,"ok":380,"rps":63,"p99_us":5706382,"fail":0},{"conc":256,"ok":352,"rps":58,"p99_us":5628286,"fail":0},{"conc":256,"ok":267,"rps":44,"p99_us":3738118,"fail":0},{"conc":512,"ok":354,"rps":57,"p99_us":5653621,"fail":0},{"conc":512,"ok":374,"rps":62,"p99_us":5705215,"fail":0},{"conc":512,"ok":329,"rps":54,"p99_us":5350251,"fail":0},{"conc":1024,"ok":344,"rps":57,"p99_us":5600675,"fail":0},{"conc":1024,"ok":364,"rps":60,"p99_us":5482798,"fail":0},{"conc":1024,"ok":359,"rps":59,"p99_us":5197021,"fail":0},{"conc":2048,"ok":355,"rps":59,"p99_us":5157712,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":12,"rps":1,"p99_us":5932365,"fail":0},{"conc":4096,"ok":16,"rps":2,"p99_us":5953511,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":36},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":74},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":68,"certified":true,"suppressed":false},"p99_us":15465,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":68,"certified":true,"suppressed":false},"p99_us":30232,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":68,"certified":true,"suppressed":false},"p99_us":61409,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":68,"certified":true,"suppressed":false},"p99_us":124533,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":67,"certified":true,"suppressed":false},"p99_us":248972,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":67,"certified":true,"suppressed":false},"p99_us":711840,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":65,"certified":true,"suppressed":false},"p99_us":2435929,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":5520133,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":58,"certified":true,"suppressed":false},"p99_us":5628286,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":57,"certified":true,"suppressed":false},"p99_us":5653621,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":5482798,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":1,"certified":true,"suppressed":false},"p99_us":5157712,"windows":3,"clean_windows":2},{"conc":4096,"rps":{"value":2,"certified":true,"suppressed":false},"p99_us":5953511,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 1 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 412 successful gateway round trip(s) and 263451 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":412,"direct_c1_samples":263451},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.00390625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":632.30859375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":632.31640625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":632.31640625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":632.31640625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.003008508378215942,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.00390625},{"t_s":2,"rss_mib":626.71484375},{"t_s":2,"rss_mib":626.9453125},{"t_s":5,"rss_mib":627.31640625},{"t_s":6,"rss_mib":627.31640625},{"t_s":8,"rss_mib":627.75390625},{"t_s":8,"rss_mib":627.75390625},{"t_s":11,"rss_mib":628.0625},{"t_s":12,"rss_mib":628.0625},{"t_s":14,"rss_mib":628.27734375},{"t_s":15,"rss_mib":628.27734375},{"t_s":17,"rss_mib":628.55859375},{"t_s":18,"rss_mib":628.55859375},{"t_s":20,"rss_mib":628.69140625},{"t_s":21,"rss_mib":628.69140625},{"t_s":23,"rss_mib":629.0546875},{"t_s":24,"rss_mib":629.0546875},{"t_s":26,"rss_mib":629.41015625},{"t_s":27,"rss_mib":629.4140625},{"t_s":29,"rss_mib":629.7421875},{"t_s":30,"rss_mib":629.74609375},{"t_s":31,"rss_mib":629.8984375},{"t_s":33,"rss_mib":629.90234375},{"t_s":35,"rss_mib":630.26171875},{"t_s":36,"rss_mib":630.265625},{"t_s":39,"rss_mib":630.34375},{"t_s":39,"rss_mib":630.453125},{"t_s":39,"rss_mib":630.5234375},{"t_s":42,"rss_mib":630.5234375},{"t_s":45,"rss_mib":630.5234375},{"t_s":48,"rss_mib":630.5234375},{"t_s":51,"rss_mib":630.5234375},{"t_s":54,"rss_mib":630.5234375},{"t_s":57,"rss_mib":630.5234375},{"t_s":60,"rss_mib":630.5234375},{"t_s":63,"rss_mib":630.5234375},{"t_s":64,"rss_mib":630.52734375},{"t_s":66,"rss_mib":630.52734375},{"t_s":69,"rss_mib":630.52734375},{"t_s":72,"rss_mib":630.52734375},{"t_s":75,"rss_mib":630.52734375},{"t_s":77,"rss_mib":630.5390625},{"t_s":78,"rss_mib":630.5390625},{"t_s":80,"rss_mib":630.796875},{"t_s":81,"rss_mib":630.796875},{"t_s":83,"rss_mib":630.81640625},{"t_s":84,"rss_mib":630.81640625},{"t_s":86,"rss_mib":631.3828125},{"t_s":87,"rss_mib":631.3828125},{"t_s":88,"rss_mib":631.42578125},{"t_s":90,"rss_mib":631.42578125},{"t_s":93,"rss_mib":631.69140625},{"t_s":93,"rss_mib":631.69140625},{"t_s":94,"rss_mib":631.7734375},{"t_s":96,"rss_mib":631.7734375},{"t_s":97,"rss_mib":631.83203125},{"t_s":99,"rss_mib":631.83203125},{"t_s":102,"rss_mib":631.83203125},{"t_s":105,"rss_mib":631.83203125},{"t_s":108,"rss_mib":631.83203125},{"t_s":111,"rss_mib":631.83203125},{"t_s":114,"rss_mib":631.83203125},{"t_s":117,"rss_mib":631.83203125},{"t_s":120,"rss_mib":631.83203125},{"t_s":123,"rss_mib":631.83203125},{"t_s":126,"rss_mib":631.83203125},{"t_s":129,"rss_mib":631.83203125},{"t_s":132,"rss_mib":631.83203125},{"t_s":135,"rss_mib":631.83203125},{"t_s":138,"rss_mib":631.83203125},{"t_s":142,"rss_mib":631.83203125},{"t_s":144,"rss_mib":631.83984375},{"t_s":145,"rss_mib":631.83984375},{"t_s":147,"rss_mib":631.84375},{"t_s":148,"rss_mib":631.84375},{"t_s":151,"rss_mib":631.8515625},{"t_s":151,"rss_mib":631.8515625},{"t_s":154,"rss_mib":631.8515625},{"t_s":157,"rss_mib":631.890625},{"t_s":160,"rss_mib":631.890625},{"t_s":163,"rss_mib":631.890625},{"t_s":166,"rss_mib":631.890625},{"t_s":169,"rss_mib":631.890625},{"t_s":172,"rss_mib":631.890625},{"t_s":175,"rss_mib":631.890625},{"t_s":178,"rss_mib":631.890625},{"t_s":181,"rss_mib":631.890625},{"t_s":184,"rss_mib":631.890625},{"t_s":187,"rss_mib":631.890625},{"t_s":190,"rss_mib":631.890625},{"t_s":193,"rss_mib":631.890625},{"t_s":196,"rss_mib":631.890625},{"t_s":199,"rss_mib":631.890625},{"t_s":202,"rss_mib":631.890625},{"t_s":205,"rss_mib":631.890625},{"t_s":207,"rss_mib":631.9140625},{"t_s":208,"rss_mib":631.9140625},{"t_s":210,"rss_mib":631.98046875},{"t_s":211,"rss_mib":631.98828125},{"t_s":214,"rss_mib":632.1875},{"t_s":214,"rss_mib":632.1875},{"t_s":216,"rss_mib":632.3046875},{"t_s":217,"rss_mib":632.3046875},{"t_s":220,"rss_mib":632.3046875},{"t_s":220,"rss_mib":632.30859375},{"t_s":223,"rss_mib":632.30859375},{"t_s":226,"rss_mib":632.30859375},{"t_s":229,"rss_mib":632.30859375},{"t_s":232,"rss_mib":632.30859375},{"t_s":235,"rss_mib":632.3125},{"t_s":238,"rss_mib":632.3125},{"t_s":241,"rss_mib":632.3125},{"t_s":244,"rss_mib":632.3125},{"t_s":247,"rss_mib":632.3125},{"t_s":250,"rss_mib":632.3125},{"t_s":251,"rss_mib":632.31640625},{"t_s":253,"rss_mib":632.31640625},{"t_s":256,"rss_mib":632.31640625},{"t_s":259,"rss_mib":632.31640625},{"t_s":262,"rss_mib":632.31640625},{"t_s":265,"rss_mib":632.31640625},{"t_s":268,"rss_mib":632.31640625},{"t_s":271,"rss_mib":632.31640625},{"t_s":275,"rss_mib":632.31640625},{"t_s":277,"rss_mib":632.31640625},{"t_s":281,"rss_mib":632.31640625},{"t_s":284,"rss_mib":632.31640625},{"t_s":287,"rss_mib":632.31640625},{"t_s":290,"rss_mib":632.31640625},{"t_s":293,"rss_mib":632.31640625},{"t_s":296,"rss_mib":632.31640625},{"t_s":299,"rss_mib":632.31640625},{"t_s":302,"rss_mib":632.31640625},{"t_s":305,"rss_mib":632.31640625},{"t_s":308,"rss_mib":632.31640625},{"t_s":311,"rss_mib":632.31640625},{"t_s":314,"rss_mib":632.31640625},{"t_s":317,"rss_mib":632.31640625},{"t_s":320,"rss_mib":632.31640625},{"t_s":323,"rss_mib":632.31640625},{"t_s":326,"rss_mib":632.31640625},{"t_s":329,"rss_mib":632.31640625},{"t_s":332,"rss_mib":632.31640625},{"t_s":335,"rss_mib":632.31640625},{"t_s":338,"rss_mib":632.31640625},{"t_s":341,"rss_mib":632.31640625},{"t_s":344,"rss_mib":632.31640625},{"t_s":347,"rss_mib":632.31640625},{"t_s":350,"rss_mib":632.31640625},{"t_s":353,"rss_mib":632.31640625},{"t_s":356,"rss_mib":632.31640625},{"t_s":359,"rss_mib":632.31640625},{"t_s":362,"rss_mib":632.31640625}],"idle_rss_series":[{"t_s":0,"rss_mib":592.00390625},{"t_s":0,"rss_mib":592.00390625},{"t_s":0,"rss_mib":592.00390625},{"t_s":1,"rss_mib":592.00390625},{"t_s":1,"rss_mib":592.00390625},{"t_s":2,"rss_mib":592.00390625},{"t_s":2,"rss_mib":592.00390625},{"t_s":3,"rss_mib":592.00390625},{"t_s":3,"rss_mib":592.00390625},{"t_s":4,"rss_mib":592.00390625},{"t_s":5,"rss_mib":592.00390625},{"t_s":5,"rss_mib":592.00390625},{"t_s":5,"rss_mib":592.00390625},{"t_s":6,"rss_mib":592.00390625},{"t_s":6,"rss_mib":592.00390625},{"t_s":7,"rss_mib":592.00390625},{"t_s":7,"rss_mib":592.00390625},{"t_s":8,"rss_mib":592.00390625},{"t_s":8,"rss_mib":592.00390625},{"t_s":9,"rss_mib":592.00390625},{"t_s":10,"rss_mib":592.00390625},{"t_s":10,"rss_mib":592.00390625},{"t_s":10,"rss_mib":592.00390625},{"t_s":11,"rss_mib":592.00390625},{"t_s":11,"rss_mib":592.00390625},{"t_s":12,"rss_mib":592.00390625},{"t_s":12,"rss_mib":592.00390625},{"t_s":13,"rss_mib":592.00390625},{"t_s":13,"rss_mib":592.00390625},{"t_s":14,"rss_mib":592.00390625},{"t_s":15,"rss_mib":592.00390625},{"t_s":15,"rss_mib":592.00390625},{"t_s":15,"rss_mib":592.00390625},{"t_s":16,"rss_mib":592.00390625},{"t_s":16,"rss_mib":592.00390625},{"t_s":17,"rss_mib":592.00390625},{"t_s":17,"rss_mib":592.00390625},{"t_s":18,"rss_mib":592.00390625},{"t_s":18,"rss_mib":592.00390625},{"t_s":19,"rss_mib":592.00390625},{"t_s":20,"rss_mib":592.00390625},{"t_s":20,"rss_mib":592.00390625},{"t_s":20,"rss_mib":592.00390625},{"t_s":21,"rss_mib":592.00390625},{"t_s":21,"rss_mib":592.00390625},{"t_s":22,"rss_mib":592.00390625},{"t_s":22,"rss_mib":592.00390625},{"t_s":23,"rss_mib":592.00390625},{"t_s":23,"rss_mib":592.00390625},{"t_s":24,"rss_mib":592.00390625},{"t_s":25,"rss_mib":592.00390625},{"t_s":25,"rss_mib":592.00390625},{"t_s":25,"rss_mib":592.00390625},{"t_s":26,"rss_mib":592.00390625},{"t_s":26,"rss_mib":592.00390625},{"t_s":27,"rss_mib":592.00390625},{"t_s":27,"rss_mib":592.00390625},{"t_s":28,"rss_mib":592.00390625},{"t_s":28,"rss_mib":592.00390625},{"t_s":29,"rss_mib":592.00390625},{"t_s":30,"rss_mib":592.00390625},{"t_s":30,"rss_mib":592.00390625},{"t_s":30,"rss_mib":592.00390625},{"t_s":31,"rss_mib":592.00390625},{"t_s":31,"rss_mib":592.00390625},{"t_s":32,"rss_mib":592.00390625},{"t_s":32,"rss_mib":592.00390625},{"t_s":33,"rss_mib":592.00390625},{"t_s":33,"rss_mib":592.00390625},{"t_s":34,"rss_mib":592.00390625},{"t_s":35,"rss_mib":592.00390625},{"t_s":35,"rss_mib":592.00390625},{"t_s":35,"rss_mib":592.00390625},{"t_s":36,"rss_mib":592.00390625},{"t_s":36,"rss_mib":592.00390625},{"t_s":37,"rss_mib":592.00390625},{"t_s":37,"rss_mib":592.00390625},{"t_s":38,"rss_mib":592.00390625},{"t_s":38,"rss_mib":592.00390625},{"t_s":39,"rss_mib":592.00390625},{"t_s":40,"rss_mib":592.00390625},{"t_s":40,"rss_mib":592.00390625},{"t_s":40,"rss_mib":592.00390625},{"t_s":41,"rss_mib":592.00390625},{"t_s":41,"rss_mib":592.00390625},{"t_s":42,"rss_mib":592.00390625},{"t_s":42,"rss_mib":592.00390625},{"t_s":43,"rss_mib":592.00390625},{"t_s":43,"rss_mib":592.00390625},{"t_s":44,"rss_mib":592.00390625},{"t_s":45,"rss_mib":592.00390625},{"t_s":45,"rss_mib":592.00390625},{"t_s":45,"rss_mib":592.00390625},{"t_s":46,"rss_mib":592.00390625},{"t_s":46,"rss_mib":592.00390625},{"t_s":47,"rss_mib":592.00390625},{"t_s":47,"rss_mib":592.00390625},{"t_s":48,"rss_mib":592.00390625},{"t_s":48,"rss_mib":592.00390625},{"t_s":49,"rss_mib":592.00390625},{"t_s":50,"rss_mib":592.00390625},{"t_s":50,"rss_mib":592.00390625},{"t_s":50,"rss_mib":592.00390625},{"t_s":51,"rss_mib":592.00390625},{"t_s":51,"rss_mib":592.00390625},{"t_s":52,"rss_mib":592.00390625},{"t_s":52,"rss_mib":592.00390625},{"t_s":53,"rss_mib":592.00390625},{"t_s":53,"rss_mib":592.00390625},{"t_s":54,"rss_mib":592.00390625},{"t_s":55,"rss_mib":592.00390625},{"t_s":55,"rss_mib":592.00390625},{"t_s":55,"rss_mib":592.00390625},{"t_s":56,"rss_mib":592.00390625},{"t_s":56,"rss_mib":592.00390625},{"t_s":57,"rss_mib":592.00390625},{"t_s":57,"rss_mib":592.00390625},{"t_s":58,"rss_mib":592.00390625},{"t_s":58,"rss_mib":592.00390625},{"t_s":59,"rss_mib":592.00390625},{"t_s":59,"rss_mib":592.00390625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.014288887,"cost":6.114050116,"memory":438.069857154,"streaming":0.000001801,"streams_sustained":0.000001791,"throughput":571.159337641},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}}}},"openai-responses":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"cohere":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/v2/chat","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"bedrock":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/model/gpt-4o-mini-responses/converse","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/gpt-4o-mini-responses:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":17577,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":18499,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":18527,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":18005.865102639294,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":55.537459283387626,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.250691676136018,"certified":true,"suppressed":false},"cost_window_ok":{"value":341,"certified":true,"suppressed":false},"cost_window_rps":{"value":56,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.08211143695014662,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":56,"certified":true,"suppressed":false},"concurrency":1,"p99_us":18534,"first_disqualified_conc":4,"lower_bound":false,"samples":337},{"bound_ms":100,"rps":{"value":56,"certified":true,"suppressed":false},"concurrency":1,"p99_us":18534,"first_disqualified_conc":8,"lower_bound":false,"samples":337},{"bound_ms":null,"rps":{"value":59,"certified":true,"suppressed":false},"concurrency":2048,"p99_us":5599008,"first_disqualified_conc":4096,"lower_bound":false,"samples":360}],"sweep":[{"conc":1,"ok":336,"rps":55,"p99_us":18874,"fail":0},{"conc":1,"ok":337,"rps":56,"p99_us":18579,"fail":0},{"conc":1,"ok":337,"rps":56,"p99_us":18534,"fail":0},{"conc":2,"ok":336,"rps":55,"p99_us":37173,"fail":0},{"conc":2,"ok":337,"rps":56,"p99_us":37723,"fail":0},{"conc":2,"ok":336,"rps":55,"p99_us":36992,"fail":0},{"conc":4,"ok":340,"rps":56,"p99_us":72749,"fail":0},{"conc":4,"ok":337,"rps":56,"p99_us":74075,"fail":0},{"conc":4,"ok":336,"rps":55,"p99_us":75176,"fail":0},{"conc":8,"ok":337,"rps":56,"p99_us":151487,"fail":0},{"conc":8,"ok":336,"rps":55,"p99_us":152365,"fail":0},{"conc":8,"ok":335,"rps":55,"p99_us":152062,"fail":0},{"conc":16,"ok":334,"rps":55,"p99_us":312727,"fail":0},{"conc":16,"ok":333,"rps":55,"p99_us":314032,"fail":0},{"conc":16,"ok":334,"rps":55,"p99_us":315932,"fail":0},{"conc":32,"ok":333,"rps":55,"p99_us":877576,"fail":0},{"conc":32,"ok":331,"rps":55,"p99_us":875779,"fail":0},{"conc":32,"ok":331,"rps":55,"p99_us":842713,"fail":0},{"conc":64,"ok":299,"rps":49,"p99_us":2795949,"fail":0},{"conc":64,"ok":327,"rps":54,"p99_us":2756172,"fail":0},{"conc":64,"ok":323,"rps":53,"p99_us":2786871,"fail":0},{"conc":128,"ok":312,"rps":51,"p99_us":5428831,"fail":0},{"conc":128,"ok":276,"rps":45,"p99_us":2724309,"fail":0},{"conc":128,"ok":308,"rps":51,"p99_us":5529698,"fail":0},{"conc":256,"ok":226,"rps":37,"p99_us":5400022,"fail":0},{"conc":256,"ok":310,"rps":51,"p99_us":5484859,"fail":0},{"conc":256,"ok":256,"rps":42,"p99_us":4650293,"fail":0},{"conc":512,"ok":267,"rps":44,"p99_us":5815824,"fail":0},{"conc":512,"ok":285,"rps":47,"p99_us":5277016,"fail":0},{"conc":512,"ok":322,"rps":53,"p99_us":4915954,"fail":0},{"conc":1024,"ok":320,"rps":52,"p99_us":4668476,"fail":0},{"conc":1024,"ok":346,"rps":57,"p99_us":5273293,"fail":0},{"conc":1024,"ok":311,"rps":51,"p99_us":5601865,"fail":0},{"conc":2048,"ok":360,"rps":59,"p99_us":5599008,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":2048,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":18579,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":37173,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":56,"certified":true,"suppressed":false},"p99_us":74075,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":152062,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":314032,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":55,"certified":true,"suppressed":false},"p99_us":875779,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":53,"certified":true,"suppressed":false},"p99_us":2786871,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":51,"certified":true,"suppressed":false},"p99_us":5428831,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":42,"certified":true,"suppressed":false},"p99_us":5400022,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":47,"certified":true,"suppressed":false},"p99_us":5277016,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":52,"certified":true,"suppressed":false},"p99_us":5273293,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":5599008,"windows":3,"clean_windows":1},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 0 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 338 successful gateway round trip(s) and 262004 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":338,"direct_c1_samples":262004},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.40625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":674.6484375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":306,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.40625},{"t_s":2,"rss_mib":669.5234375},{"t_s":2,"rss_mib":669.5234375},{"t_s":5,"rss_mib":669.546875},{"t_s":6,"rss_mib":669.55078125},{"t_s":8,"rss_mib":670.1796875},{"t_s":9,"rss_mib":670.1796875},{"t_s":12,"rss_mib":670.6875},{"t_s":12,"rss_mib":670.80859375},{"t_s":12,"rss_mib":670.8125},{"t_s":15,"rss_mib":670.81640625},{"t_s":17,"rss_mib":670.87109375},{"t_s":18,"rss_mib":670.8984375},{"t_s":18,"rss_mib":671.1171875},{"t_s":21,"rss_mib":671.12109375},{"t_s":22,"rss_mib":671.46484375},{"t_s":24,"rss_mib":671.46484375},{"t_s":25,"rss_mib":671.625},{"t_s":27,"rss_mib":671.625},{"t_s":30,"rss_mib":671.66015625},{"t_s":30,"rss_mib":671.66015625},{"t_s":32,"rss_mib":671.671875},{"t_s":33,"rss_mib":671.671875},{"t_s":35,"rss_mib":672.0703125},{"t_s":36,"rss_mib":672.0703125},{"t_s":38,"rss_mib":672.14453125},{"t_s":39,"rss_mib":672.14453125},{"t_s":40,"rss_mib":672.265625},{"t_s":42,"rss_mib":672.265625},{"t_s":45,"rss_mib":672.265625},{"t_s":48,"rss_mib":672.2734375},{"t_s":48,"rss_mib":672.2734375},{"t_s":51,"rss_mib":672.2734375},{"t_s":54,"rss_mib":672.2734375},{"t_s":57,"rss_mib":672.28125},{"t_s":57,"rss_mib":672.28125},{"t_s":59,"rss_mib":672.28515625},{"t_s":61,"rss_mib":672.28515625},{"t_s":63,"rss_mib":672.29296875},{"t_s":64,"rss_mib":672.29296875},{"t_s":67,"rss_mib":672.29296875},{"t_s":68,"rss_mib":672.359375},{"t_s":70,"rss_mib":672.359375},{"t_s":70,"rss_mib":672.36328125},{"t_s":73,"rss_mib":672.36328125},{"t_s":75,"rss_mib":672.40625},{"t_s":76,"rss_mib":672.40625},{"t_s":78,"rss_mib":672.53125},{"t_s":79,"rss_mib":672.53125},{"t_s":82,"rss_mib":672.83984375},{"t_s":82,"rss_mib":672.83984375},{"t_s":85,"rss_mib":672.921875},{"t_s":85,"rss_mib":673.01171875},{"t_s":87,"rss_mib":673.05078125},{"t_s":88,"rss_mib":673.0546875},{"t_s":88,"rss_mib":673.1640625},{"t_s":91,"rss_mib":673.1640625},{"t_s":93,"rss_mib":673.5},{"t_s":94,"rss_mib":673.5},{"t_s":95,"rss_mib":673.83203125},{"t_s":97,"rss_mib":673.83203125},{"t_s":100,"rss_mib":673.96875},{"t_s":100,"rss_mib":674.0078125},{"t_s":102,"rss_mib":674.015625},{"t_s":103,"rss_mib":674.015625},{"t_s":103,"rss_mib":674.01953125},{"t_s":106,"rss_mib":674.01953125},{"t_s":109,"rss_mib":674.01953125},{"t_s":112,"rss_mib":674.01953125},{"t_s":115,"rss_mib":674.01953125},{"t_s":118,"rss_mib":674.01953125},{"t_s":122,"rss_mib":674.01953125},{"t_s":125,"rss_mib":674.01953125},{"t_s":128,"rss_mib":674.01953125},{"t_s":130,"rss_mib":674.0234375},{"t_s":131,"rss_mib":674.0234375},{"t_s":132,"rss_mib":674.03515625},{"t_s":134,"rss_mib":674.03515625},{"t_s":137,"rss_mib":674.03515625},{"t_s":140,"rss_mib":674.03515625},{"t_s":143,"rss_mib":674.03515625},{"t_s":146,"rss_mib":674.03515625},{"t_s":149,"rss_mib":674.03515625},{"t_s":152,"rss_mib":674.03515625},{"t_s":155,"rss_mib":674.1171875},{"t_s":155,"rss_mib":674.1171875},{"t_s":158,"rss_mib":674.125},{"t_s":158,"rss_mib":674.140625},{"t_s":158,"rss_mib":674.20703125},{"t_s":161,"rss_mib":674.21484375},{"t_s":164,"rss_mib":674.63671875},{"t_s":164,"rss_mib":674.63671875},{"t_s":167,"rss_mib":674.63671875},{"t_s":170,"rss_mib":674.63671875},{"t_s":173,"rss_mib":674.640625},{"t_s":173,"rss_mib":674.640625},{"t_s":176,"rss_mib":674.640625},{"t_s":179,"rss_mib":674.640625},{"t_s":183,"rss_mib":674.640625},{"t_s":186,"rss_mib":674.640625},{"t_s":189,"rss_mib":674.640625},{"t_s":192,"rss_mib":674.640625},{"t_s":195,"rss_mib":674.640625},{"t_s":198,"rss_mib":674.640625},{"t_s":201,"rss_mib":674.640625},{"t_s":201,"rss_mib":674.64453125},{"t_s":204,"rss_mib":674.64453125},{"t_s":207,"rss_mib":674.64453125},{"t_s":210,"rss_mib":674.64453125},{"t_s":213,"rss_mib":674.64453125},{"t_s":214,"rss_mib":674.6484375},{"t_s":216,"rss_mib":674.6484375},{"t_s":219,"rss_mib":674.6484375},{"t_s":222,"rss_mib":674.6484375},{"t_s":225,"rss_mib":674.6484375},{"t_s":228,"rss_mib":674.6484375},{"t_s":231,"rss_mib":674.6484375},{"t_s":234,"rss_mib":674.6484375},{"t_s":237,"rss_mib":674.6484375},{"t_s":240,"rss_mib":674.6484375},{"t_s":244,"rss_mib":674.6484375},{"t_s":247,"rss_mib":674.6484375},{"t_s":250,"rss_mib":674.6484375},{"t_s":253,"rss_mib":674.6484375},{"t_s":256,"rss_mib":674.6484375},{"t_s":259,"rss_mib":674.6484375},{"t_s":262,"rss_mib":674.6484375},{"t_s":265,"rss_mib":674.6484375},{"t_s":268,"rss_mib":674.6484375},{"t_s":271,"rss_mib":674.6484375},{"t_s":274,"rss_mib":674.6484375},{"t_s":277,"rss_mib":674.6484375},{"t_s":280,"rss_mib":674.6484375},{"t_s":283,"rss_mib":674.6484375},{"t_s":286,"rss_mib":674.6484375},{"t_s":289,"rss_mib":674.6484375},{"t_s":292,"rss_mib":674.6484375},{"t_s":295,"rss_mib":674.6484375},{"t_s":299,"rss_mib":674.6484375},{"t_s":301,"rss_mib":674.6484375},{"t_s":305,"rss_mib":674.6484375},{"t_s":308,"rss_mib":674.6484375},{"t_s":311,"rss_mib":674.6484375},{"t_s":314,"rss_mib":674.6484375},{"t_s":317,"rss_mib":674.6484375},{"t_s":320,"rss_mib":674.6484375},{"t_s":323,"rss_mib":674.6484375},{"t_s":326,"rss_mib":674.6484375},{"t_s":329,"rss_mib":674.6484375},{"t_s":332,"rss_mib":674.6484375},{"t_s":335,"rss_mib":674.6484375},{"t_s":338,"rss_mib":674.6484375},{"t_s":341,"rss_mib":674.6484375},{"t_s":344,"rss_mib":674.6484375},{"t_s":347,"rss_mib":674.6484375},{"t_s":350,"rss_mib":674.6484375},{"t_s":353,"rss_mib":674.6484375},{"t_s":356,"rss_mib":674.6484375},{"t_s":359,"rss_mib":674.6484375},{"t_s":363,"rss_mib":674.6484375},{"t_s":365,"rss_mib":674.6484375}],"idle_rss_series":[{"t_s":0,"rss_mib":592.40625},{"t_s":0,"rss_mib":592.40625},{"t_s":0,"rss_mib":592.40625},{"t_s":1,"rss_mib":592.40625},{"t_s":1,"rss_mib":592.40625},{"t_s":2,"rss_mib":592.40625},{"t_s":2,"rss_mib":592.40625},{"t_s":3,"rss_mib":592.40625},{"t_s":3,"rss_mib":592.40625},{"t_s":4,"rss_mib":592.40625},{"t_s":5,"rss_mib":592.40625},{"t_s":5,"rss_mib":592.40625},{"t_s":5,"rss_mib":592.40625},{"t_s":6,"rss_mib":592.40625},{"t_s":6,"rss_mib":592.40625},{"t_s":7,"rss_mib":592.40625},{"t_s":7,"rss_mib":592.40625},{"t_s":8,"rss_mib":592.40625},{"t_s":8,"rss_mib":592.40625},{"t_s":9,"rss_mib":592.40625},{"t_s":10,"rss_mib":592.40625},{"t_s":10,"rss_mib":592.40625},{"t_s":10,"rss_mib":592.40625},{"t_s":11,"rss_mib":592.40625},{"t_s":11,"rss_mib":592.40625},{"t_s":12,"rss_mib":592.40625},{"t_s":12,"rss_mib":592.40625},{"t_s":13,"rss_mib":592.40625},{"t_s":13,"rss_mib":592.40625},{"t_s":14,"rss_mib":592.40625},{"t_s":15,"rss_mib":592.40625},{"t_s":15,"rss_mib":592.40625},{"t_s":15,"rss_mib":592.40625},{"t_s":16,"rss_mib":592.40625},{"t_s":16,"rss_mib":592.40625},{"t_s":17,"rss_mib":592.40625},{"t_s":17,"rss_mib":592.40625},{"t_s":18,"rss_mib":592.40625},{"t_s":18,"rss_mib":592.40625},{"t_s":19,"rss_mib":592.40625},{"t_s":20,"rss_mib":592.40625},{"t_s":20,"rss_mib":592.40625},{"t_s":20,"rss_mib":592.40625},{"t_s":21,"rss_mib":592.40625},{"t_s":21,"rss_mib":592.40625},{"t_s":22,"rss_mib":592.40625},{"t_s":22,"rss_mib":592.40625},{"t_s":23,"rss_mib":592.40625},{"t_s":23,"rss_mib":592.40625},{"t_s":24,"rss_mib":592.40625},{"t_s":25,"rss_mib":592.40625},{"t_s":25,"rss_mib":592.40625},{"t_s":25,"rss_mib":592.40625},{"t_s":26,"rss_mib":592.40625},{"t_s":26,"rss_mib":592.40625},{"t_s":27,"rss_mib":592.40625},{"t_s":27,"rss_mib":592.40625},{"t_s":28,"rss_mib":592.40625},{"t_s":28,"rss_mib":592.40625},{"t_s":29,"rss_mib":592.40625},{"t_s":30,"rss_mib":592.40625},{"t_s":30,"rss_mib":592.40625},{"t_s":30,"rss_mib":592.40625},{"t_s":31,"rss_mib":592.40625},{"t_s":31,"rss_mib":592.40625},{"t_s":32,"rss_mib":592.40625},{"t_s":32,"rss_mib":592.40625},{"t_s":33,"rss_mib":592.40625},{"t_s":33,"rss_mib":592.40625},{"t_s":34,"rss_mib":592.40625},{"t_s":35,"rss_mib":592.40625},{"t_s":35,"rss_mib":592.40625},{"t_s":35,"rss_mib":592.40625},{"t_s":36,"rss_mib":592.40625},{"t_s":36,"rss_mib":592.40625},{"t_s":37,"rss_mib":592.40625},{"t_s":37,"rss_mib":592.40625},{"t_s":38,"rss_mib":592.40625},{"t_s":38,"rss_mib":592.40625},{"t_s":39,"rss_mib":592.40625},{"t_s":40,"rss_mib":592.40625},{"t_s":40,"rss_mib":592.40625},{"t_s":40,"rss_mib":592.40625},{"t_s":41,"rss_mib":592.40625},{"t_s":41,"rss_mib":592.40625},{"t_s":42,"rss_mib":592.40625},{"t_s":42,"rss_mib":592.40625},{"t_s":43,"rss_mib":592.40625},{"t_s":43,"rss_mib":592.40625},{"t_s":44,"rss_mib":592.40625},{"t_s":45,"rss_mib":592.40625},{"t_s":45,"rss_mib":592.40625},{"t_s":45,"rss_mib":592.40625},{"t_s":46,"rss_mib":592.40625},{"t_s":46,"rss_mib":592.40625},{"t_s":47,"rss_mib":592.40625},{"t_s":47,"rss_mib":592.40625},{"t_s":48,"rss_mib":592.40625},{"t_s":48,"rss_mib":592.40625},{"t_s":49,"rss_mib":592.40625},{"t_s":50,"rss_mib":592.40625},{"t_s":50,"rss_mib":592.40625},{"t_s":50,"rss_mib":592.40625},{"t_s":51,"rss_mib":592.40625},{"t_s":51,"rss_mib":592.40625},{"t_s":52,"rss_mib":592.40625},{"t_s":52,"rss_mib":592.40625},{"t_s":53,"rss_mib":592.40625},{"t_s":53,"rss_mib":592.40625},{"t_s":54,"rss_mib":592.40625},{"t_s":55,"rss_mib":592.40625},{"t_s":55,"rss_mib":592.40625},{"t_s":55,"rss_mib":592.40625},{"t_s":56,"rss_mib":592.40625},{"t_s":56,"rss_mib":592.40625},{"t_s":57,"rss_mib":592.40625},{"t_s":57,"rss_mib":592.40625},{"t_s":58,"rss_mib":592.40625},{"t_s":58,"rss_mib":592.40625},{"t_s":59,"rss_mib":592.40625},{"t_s":59,"rss_mib":592.40625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.011284302,"cost":6.136360277,"memory":441.49543176,"streaming":0.00000187,"streams_sustained":0.00000176,"throughput":486.695262169},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c2048.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c2048.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}},"anthropic":{"served":true,"reason":null,"status":"","path":"/v1/messages","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":13287,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":14010,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":14038,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":13738.738738738739,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":72.78688524590164,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25008979839096346,"certified":true,"suppressed":false},"cost_window_ok":{"value":444,"certified":true,"suppressed":false},"cost_window_rps":{"value":73,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.07207207207207207,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":74,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13937,"first_disqualified_conc":4,"lower_bound":false,"samples":450},{"bound_ms":100,"rps":{"value":74,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13937,"first_disqualified_conc":8,"lower_bound":false,"samples":450},{"bound_ms":null,"rps":{"value":74,"certified":true,"suppressed":false},"concurrency":1,"p99_us":13937,"first_disqualified_conc":8192,"lower_bound":false,"samples":450}],"sweep":[{"conc":1,"ok":447,"rps":74,"p99_us":14091,"fail":0},{"conc":1,"ok":447,"rps":74,"p99_us":14120,"fail":0},{"conc":1,"ok":450,"rps":74,"p99_us":13937,"fail":0},{"conc":2,"ok":448,"rps":74,"p99_us":28609,"fail":0},{"conc":2,"ok":450,"rps":74,"p99_us":27421,"fail":0},{"conc":2,"ok":450,"rps":74,"p99_us":27405,"fail":0},{"conc":4,"ok":446,"rps":74,"p99_us":58790,"fail":0},{"conc":4,"ok":447,"rps":74,"p99_us":57827,"fail":0},{"conc":4,"ok":449,"rps":74,"p99_us":55786,"fail":0},{"conc":8,"ok":449,"rps":74,"p99_us":113204,"fail":0},{"conc":8,"ok":448,"rps":74,"p99_us":114109,"fail":0},{"conc":8,"ok":447,"rps":74,"p99_us":113840,"fail":0},{"conc":16,"ok":446,"rps":74,"p99_us":228159,"fail":0},{"conc":16,"ok":444,"rps":73,"p99_us":227210,"fail":0},{"conc":16,"ok":445,"rps":74,"p99_us":232596,"fail":0},{"conc":32,"ok":445,"rps":74,"p99_us":648967,"fail":0},{"conc":32,"ok":446,"rps":74,"p99_us":619359,"fail":0},{"conc":32,"ok":448,"rps":74,"p99_us":627825,"fail":0},{"conc":64,"ok":437,"rps":72,"p99_us":1392342,"fail":0},{"conc":64,"ok":423,"rps":70,"p99_us":1302359,"fail":0},{"conc":64,"ok":439,"rps":73,"p99_us":2153360,"fail":0},{"conc":128,"ok":428,"rps":71,"p99_us":5570314,"fail":0},{"conc":128,"ok":425,"rps":70,"p99_us":5604978,"fail":0},{"conc":128,"ok":384,"rps":63,"p99_us":2119019,"fail":0},{"conc":256,"ok":429,"rps":71,"p99_us":5565656,"fail":0},{"conc":256,"ok":347,"rps":57,"p99_us":3560498,"fail":0},{"conc":256,"ok":407,"rps":67,"p99_us":5488968,"fail":0},{"conc":512,"ok":427,"rps":71,"p99_us":5388983,"fail":0},{"conc":512,"ok":330,"rps":54,"p99_us":5109680,"fail":0},{"conc":512,"ok":325,"rps":54,"p99_us":4878613,"fail":0},{"conc":1024,"ok":361,"rps":60,"p99_us":5102098,"fail":0},{"conc":1024,"ok":427,"rps":71,"p99_us":5632694,"fail":0},{"conc":1024,"ok":357,"rps":59,"p99_us":5072170,"fail":0},{"conc":2048,"ok":361,"rps":60,"p99_us":5033092,"fail":0},{"conc":2048,"ok":227,"rps":37,"p99_us":5713964,"fail":0},{"conc":2048,"ok":240,"rps":39,"p99_us":5717164,"fail":0},{"conc":4096,"ok":269,"rps":44,"p99_us":5723433,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":5260}],"rungs":[{"conc":1,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":14091,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":27421,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":57827,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":113840,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":228159,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":74,"certified":true,"suppressed":false},"p99_us":627825,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":72,"certified":true,"suppressed":false},"p99_us":1392342,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":70,"certified":true,"suppressed":false},"p99_us":5570314,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":67,"certified":true,"suppressed":false},"p99_us":5488968,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":54,"certified":true,"suppressed":false},"p99_us":5109680,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":5102098,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":39,"certified":true,"suppressed":false},"p99_us":5713964,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":44,"certified":true,"suppressed":false},"p99_us":5723433,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 1 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 449 successful gateway round trip(s) and 260854 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":449,"direct_c1_samples":260854},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.65234375,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":661.2265625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":303,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.65234375},{"t_s":2,"rss_mib":661.19921875},{"t_s":2,"rss_mib":661.19921875},{"t_s":3,"rss_mib":661.203125},{"t_s":6,"rss_mib":661.203125},{"t_s":6,"rss_mib":661.20703125},{"t_s":8,"rss_mib":661.20703125},{"t_s":12,"rss_mib":661.20703125},{"t_s":14,"rss_mib":661.21484375},{"t_s":15,"rss_mib":661.21484375},{"t_s":18,"rss_mib":661.21484375},{"t_s":21,"rss_mib":661.21484375},{"t_s":24,"rss_mib":661.21484375},{"t_s":26,"rss_mib":661.21875},{"t_s":27,"rss_mib":661.21875},{"t_s":30,"rss_mib":661.21875},{"t_s":31,"rss_mib":661.22265625},{"t_s":33,"rss_mib":661.22265625},{"t_s":36,"rss_mib":661.22265625},{"t_s":39,"rss_mib":661.22265625},{"t_s":42,"rss_mib":661.22265625},{"t_s":45,"rss_mib":661.22265625},{"t_s":48,"rss_mib":661.22265625},{"t_s":51,"rss_mib":661.22265625},{"t_s":54,"rss_mib":661.22265625},{"t_s":57,"rss_mib":661.22265625},{"t_s":60,"rss_mib":661.22265625},{"t_s":61,"rss_mib":661.2265625},{"t_s":63,"rss_mib":661.2265625},{"t_s":66,"rss_mib":661.2265625},{"t_s":69,"rss_mib":661.2265625},{"t_s":72,"rss_mib":661.2265625},{"t_s":75,"rss_mib":661.2265625},{"t_s":78,"rss_mib":661.2265625},{"t_s":81,"rss_mib":661.2265625},{"t_s":84,"rss_mib":661.2265625},{"t_s":87,"rss_mib":661.2265625},{"t_s":90,"rss_mib":661.2265625},{"t_s":93,"rss_mib":661.2265625},{"t_s":96,"rss_mib":661.2265625},{"t_s":99,"rss_mib":661.2265625},{"t_s":102,"rss_mib":661.2265625},{"t_s":105,"rss_mib":661.2265625},{"t_s":108,"rss_mib":661.2265625},{"t_s":111,"rss_mib":661.2265625},{"t_s":114,"rss_mib":661.2265625},{"t_s":118,"rss_mib":661.2265625},{"t_s":120,"rss_mib":661.2265625},{"t_s":124,"rss_mib":661.2265625},{"t_s":127,"rss_mib":661.2265625},{"t_s":130,"rss_mib":661.2265625},{"t_s":133,"rss_mib":661.2265625},{"t_s":136,"rss_mib":661.2265625},{"t_s":139,"rss_mib":661.2265625},{"t_s":142,"rss_mib":661.2265625},{"t_s":145,"rss_mib":661.2265625},{"t_s":148,"rss_mib":661.2265625},{"t_s":151,"rss_mib":661.2265625},{"t_s":154,"rss_mib":661.2265625},{"t_s":157,"rss_mib":661.2265625},{"t_s":160,"rss_mib":661.2265625},{"t_s":163,"rss_mib":661.2265625},{"t_s":166,"rss_mib":661.2265625},{"t_s":169,"rss_mib":661.2265625},{"t_s":172,"rss_mib":661.2265625},{"t_s":175,"rss_mib":661.2265625},{"t_s":178,"rss_mib":661.2265625},{"t_s":181,"rss_mib":661.2265625},{"t_s":184,"rss_mib":661.2265625},{"t_s":187,"rss_mib":661.2265625},{"t_s":190,"rss_mib":661.2265625},{"t_s":193,"rss_mib":661.2265625},{"t_s":196,"rss_mib":661.2265625},{"t_s":199,"rss_mib":661.2265625},{"t_s":202,"rss_mib":661.2265625},{"t_s":205,"rss_mib":661.2265625},{"t_s":208,"rss_mib":661.2265625},{"t_s":211,"rss_mib":661.2265625},{"t_s":214,"rss_mib":661.2265625},{"t_s":217,"rss_mib":661.2265625},{"t_s":220,"rss_mib":661.2265625},{"t_s":223,"rss_mib":661.2265625},{"t_s":226,"rss_mib":661.2265625},{"t_s":229,"rss_mib":661.2265625},{"t_s":232,"rss_mib":661.2265625},{"t_s":236,"rss_mib":661.2265625},{"t_s":238,"rss_mib":661.2265625},{"t_s":242,"rss_mib":661.2265625},{"t_s":245,"rss_mib":661.2265625},{"t_s":248,"rss_mib":661.2265625},{"t_s":251,"rss_mib":661.2265625},{"t_s":254,"rss_mib":661.2265625},{"t_s":257,"rss_mib":661.2265625},{"t_s":260,"rss_mib":661.2265625},{"t_s":263,"rss_mib":661.2265625},{"t_s":266,"rss_mib":661.2265625},{"t_s":269,"rss_mib":661.2265625},{"t_s":272,"rss_mib":661.2265625},{"t_s":275,"rss_mib":661.2265625},{"t_s":278,"rss_mib":661.2265625},{"t_s":281,"rss_mib":661.2265625},{"t_s":284,"rss_mib":661.2265625},{"t_s":287,"rss_mib":661.2265625},{"t_s":290,"rss_mib":661.2265625},{"t_s":293,"rss_mib":661.2265625},{"t_s":296,"rss_mib":661.2265625},{"t_s":299,"rss_mib":661.2265625},{"t_s":302,"rss_mib":661.2265625},{"t_s":305,"rss_mib":661.2265625},{"t_s":308,"rss_mib":661.2265625},{"t_s":311,"rss_mib":661.2265625},{"t_s":314,"rss_mib":661.2265625},{"t_s":317,"rss_mib":661.2265625},{"t_s":320,"rss_mib":661.2265625},{"t_s":323,"rss_mib":661.2265625},{"t_s":326,"rss_mib":661.2265625},{"t_s":329,"rss_mib":661.2265625},{"t_s":332,"rss_mib":661.2265625},{"t_s":335,"rss_mib":661.2265625},{"t_s":338,"rss_mib":661.2265625},{"t_s":341,"rss_mib":661.2265625},{"t_s":344,"rss_mib":661.2265625},{"t_s":347,"rss_mib":661.2265625},{"t_s":350,"rss_mib":661.2265625},{"t_s":353,"rss_mib":661.2265625},{"t_s":356,"rss_mib":661.2265625},{"t_s":360,"rss_mib":661.2265625},{"t_s":362,"rss_mib":661.2265625}],"idle_rss_series":[{"t_s":0,"rss_mib":591.65234375},{"t_s":0,"rss_mib":591.65234375},{"t_s":0,"rss_mib":591.65234375},{"t_s":1,"rss_mib":591.65234375},{"t_s":1,"rss_mib":591.65234375},{"t_s":2,"rss_mib":591.65234375},{"t_s":2,"rss_mib":591.65234375},{"t_s":3,"rss_mib":591.65234375},{"t_s":3,"rss_mib":591.65234375},{"t_s":4,"rss_mib":591.65234375},{"t_s":5,"rss_mib":591.65234375},{"t_s":5,"rss_mib":591.65234375},{"t_s":5,"rss_mib":591.65234375},{"t_s":6,"rss_mib":591.65234375},{"t_s":6,"rss_mib":591.65234375},{"t_s":7,"rss_mib":591.65234375},{"t_s":7,"rss_mib":591.65234375},{"t_s":8,"rss_mib":591.65234375},{"t_s":8,"rss_mib":591.65234375},{"t_s":9,"rss_mib":591.65234375},{"t_s":10,"rss_mib":591.65234375},{"t_s":10,"rss_mib":591.65234375},{"t_s":10,"rss_mib":591.65234375},{"t_s":11,"rss_mib":591.65234375},{"t_s":11,"rss_mib":591.65234375},{"t_s":12,"rss_mib":591.65234375},{"t_s":12,"rss_mib":591.65234375},{"t_s":13,"rss_mib":591.65234375},{"t_s":13,"rss_mib":591.65234375},{"t_s":14,"rss_mib":591.65234375},{"t_s":15,"rss_mib":591.65234375},{"t_s":15,"rss_mib":591.65234375},{"t_s":15,"rss_mib":591.65234375},{"t_s":16,"rss_mib":591.65234375},{"t_s":16,"rss_mib":591.65234375},{"t_s":17,"rss_mib":591.65234375},{"t_s":17,"rss_mib":591.65234375},{"t_s":18,"rss_mib":591.65234375},{"t_s":18,"rss_mib":591.65234375},{"t_s":19,"rss_mib":591.65234375},{"t_s":20,"rss_mib":591.65234375},{"t_s":20,"rss_mib":591.65234375},{"t_s":20,"rss_mib":591.65234375},{"t_s":21,"rss_mib":591.65234375},{"t_s":21,"rss_mib":591.65234375},{"t_s":22,"rss_mib":591.65234375},{"t_s":22,"rss_mib":591.65234375},{"t_s":23,"rss_mib":591.65234375},{"t_s":23,"rss_mib":591.65234375},{"t_s":24,"rss_mib":591.65234375},{"t_s":25,"rss_mib":591.65234375},{"t_s":25,"rss_mib":591.65234375},{"t_s":25,"rss_mib":591.65234375},{"t_s":26,"rss_mib":591.65234375},{"t_s":26,"rss_mib":591.65234375},{"t_s":27,"rss_mib":591.65234375},{"t_s":27,"rss_mib":591.65234375},{"t_s":28,"rss_mib":591.65234375},{"t_s":28,"rss_mib":591.65234375},{"t_s":29,"rss_mib":591.65234375},{"t_s":30,"rss_mib":591.65234375},{"t_s":30,"rss_mib":591.65234375},{"t_s":30,"rss_mib":591.65234375},{"t_s":31,"rss_mib":591.65234375},{"t_s":31,"rss_mib":591.65234375},{"t_s":32,"rss_mib":591.65234375},{"t_s":32,"rss_mib":591.65234375},{"t_s":33,"rss_mib":591.65234375},{"t_s":33,"rss_mib":591.65234375},{"t_s":34,"rss_mib":591.65234375},{"t_s":35,"rss_mib":591.65234375},{"t_s":35,"rss_mib":591.65234375},{"t_s":35,"rss_mib":591.65234375},{"t_s":36,"rss_mib":591.65234375},{"t_s":36,"rss_mib":591.65234375},{"t_s":37,"rss_mib":591.65234375},{"t_s":37,"rss_mib":591.65234375},{"t_s":38,"rss_mib":591.65234375},{"t_s":38,"rss_mib":591.65234375},{"t_s":39,"rss_mib":591.65234375},{"t_s":40,"rss_mib":591.65234375},{"t_s":40,"rss_mib":591.65234375},{"t_s":40,"rss_mib":591.65234375},{"t_s":41,"rss_mib":591.65234375},{"t_s":41,"rss_mib":591.65234375},{"t_s":42,"rss_mib":591.65234375},{"t_s":42,"rss_mib":591.65234375},{"t_s":43,"rss_mib":591.65234375},{"t_s":43,"rss_mib":591.65234375},{"t_s":44,"rss_mib":591.65234375},{"t_s":45,"rss_mib":591.65234375},{"t_s":45,"rss_mib":591.65234375},{"t_s":45,"rss_mib":591.65234375},{"t_s":46,"rss_mib":591.65234375},{"t_s":46,"rss_mib":591.65234375},{"t_s":47,"rss_mib":591.65234375},{"t_s":47,"rss_mib":591.65234375},{"t_s":48,"rss_mib":591.65234375},{"t_s":48,"rss_mib":591.65234375},{"t_s":49,"rss_mib":591.65234375},{"t_s":50,"rss_mib":591.65234375},{"t_s":50,"rss_mib":591.65234375},{"t_s":50,"rss_mib":591.65234375},{"t_s":51,"rss_mib":591.65234375},{"t_s":51,"rss_mib":591.65234375},{"t_s":52,"rss_mib":591.65234375},{"t_s":52,"rss_mib":591.65234375},{"t_s":53,"rss_mib":591.65234375},{"t_s":53,"rss_mib":591.65234375},{"t_s":54,"rss_mib":591.65234375},{"t_s":55,"rss_mib":591.65234375},{"t_s":55,"rss_mib":591.65234375},{"t_s":55,"rss_mib":591.65234375},{"t_s":56,"rss_mib":591.65234375},{"t_s":56,"rss_mib":591.65234375},{"t_s":57,"rss_mib":591.65234375},{"t_s":57,"rss_mib":591.65234375},{"t_s":58,"rss_mib":591.65234375},{"t_s":58,"rss_mib":591.65234375},{"t_s":59,"rss_mib":591.65234375},{"t_s":59,"rss_mib":591.65234375}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.016989271,"cost":6.110977409,"memory":438.502132933,"streaming":0.000003171,"streams_sustained":0.000001768,"throughput":564.379953277},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10726,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":11474,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":11502,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10799.289520426288,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":92.59868421052632,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2502330796843856,"certified":true,"suppressed":false},"cost_window_ok":{"value":563,"certified":true,"suppressed":false},"cost_window_rps":{"value":93,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.05328596802841918,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":92,"certified":true,"suppressed":false},"concurrency":1,"p99_us":11558,"first_disqualified_conc":8,"lower_bound":false,"samples":556},{"bound_ms":100,"rps":{"value":92,"certified":true,"suppressed":false},"concurrency":1,"p99_us":11558,"first_disqualified_conc":16,"lower_bound":false,"samples":556},{"bound_ms":null,"rps":{"value":93,"certified":true,"suppressed":false},"concurrency":16,"p99_us":183258,"first_disqualified_conc":16384,"lower_bound":false,"samples":559}],"sweep":[{"conc":1,"ok":549,"rps":91,"p99_us":11579,"fail":0},{"conc":1,"ok":553,"rps":92,"p99_us":11563,"fail":0},{"conc":1,"ok":556,"rps":92,"p99_us":11558,"fail":0},{"conc":2,"ok":558,"rps":92,"p99_us":22473,"fail":0},{"conc":2,"ok":554,"rps":92,"p99_us":22721,"fail":0},{"conc":2,"ok":554,"rps":92,"p99_us":22545,"fail":0},{"conc":4,"ok":549,"rps":91,"p99_us":44716,"fail":0},{"conc":4,"ok":557,"rps":92,"p99_us":45546,"fail":0},{"conc":4,"ok":553,"rps":92,"p99_us":45970,"fail":0},{"conc":8,"ok":553,"rps":92,"p99_us":93624,"fail":0},{"conc":8,"ok":552,"rps":91,"p99_us":93692,"fail":0},{"conc":8,"ok":553,"rps":92,"p99_us":94238,"fail":0},{"conc":16,"ok":554,"rps":92,"p99_us":194426,"fail":0},{"conc":16,"ok":558,"rps":92,"p99_us":186433,"fail":0},{"conc":16,"ok":559,"rps":93,"p99_us":183258,"fail":0},{"conc":32,"ok":558,"rps":92,"p99_us":496491,"fail":0},{"conc":32,"ok":557,"rps":92,"p99_us":514679,"fail":0},{"conc":32,"ok":556,"rps":92,"p99_us":407429,"fail":0},{"conc":64,"ok":544,"rps":90,"p99_us":1657904,"fail":0},{"conc":64,"ok":501,"rps":83,"p99_us":1688112,"fail":0},{"conc":64,"ok":542,"rps":90,"p99_us":1725589,"fail":0},{"conc":128,"ok":516,"rps":85,"p99_us":5411768,"fail":0},{"conc":128,"ok":512,"rps":85,"p99_us":1847733,"fail":0},{"conc":128,"ok":523,"rps":87,"p99_us":5502568,"fail":0},{"conc":256,"ok":490,"rps":81,"p99_us":5532833,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":3202882,"fail":0},{"conc":256,"ok":526,"rps":87,"p99_us":5518065,"fail":0},{"conc":512,"ok":363,"rps":60,"p99_us":4736588,"fail":0},{"conc":512,"ok":323,"rps":53,"p99_us":5137110,"fail":0},{"conc":512,"ok":325,"rps":54,"p99_us":5172042,"fail":0},{"conc":1024,"ok":366,"rps":60,"p99_us":5312283,"fail":0},{"conc":1024,"ok":371,"rps":61,"p99_us":5141173,"fail":0},{"conc":1024,"ok":363,"rps":60,"p99_us":4337892,"fail":0},{"conc":2048,"ok":371,"rps":61,"p99_us":4698832,"fail":0},{"conc":2048,"ok":372,"rps":61,"p99_us":5550594,"fail":0},{"conc":2048,"ok":377,"rps":61,"p99_us":5090110,"fail":0},{"conc":4096,"ok":481,"rps":80,"p99_us":5605955,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":258},{"conc":8192,"ok":330,"rps":54,"p99_us":5046512,"fail":31},{"conc":8192,"ok":166,"rps":27,"p99_us":5913179,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":676},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":1129},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":324},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":214}],"rungs":[{"conc":1,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":11563,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":22545,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":45546,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":93692,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":186433,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":496491,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":90,"certified":true,"suppressed":false},"p99_us":1688112,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5411768,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5518065,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":54,"certified":true,"suppressed":false},"p99_us":5137110,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":5141173,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":5090110,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":80,"certified":true,"suppressed":false},"p99_us":5605955,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":27,"certified":true,"suppressed":false},"p99_us":5913179,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 557 successful gateway round trip(s) and 261925 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":557,"direct_c1_samples":261925},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.7890625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false},"peak_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":669.40234375,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":305,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.7890625},{"t_s":1,"rss_mib":669.40234375},{"t_s":2,"rss_mib":669.40234375},{"t_s":6,"rss_mib":669.40234375},{"t_s":9,"rss_mib":669.40234375},{"t_s":12,"rss_mib":669.40234375},{"t_s":15,"rss_mib":669.40234375},{"t_s":18,"rss_mib":669.40234375},{"t_s":21,"rss_mib":669.40234375},{"t_s":24,"rss_mib":669.40234375},{"t_s":27,"rss_mib":669.40234375},{"t_s":30,"rss_mib":669.40234375},{"t_s":33,"rss_mib":669.40234375},{"t_s":36,"rss_mib":669.40234375},{"t_s":39,"rss_mib":669.40234375},{"t_s":42,"rss_mib":669.40234375},{"t_s":45,"rss_mib":669.40234375},{"t_s":48,"rss_mib":669.40234375},{"t_s":51,"rss_mib":669.40234375},{"t_s":54,"rss_mib":669.40234375},{"t_s":57,"rss_mib":669.40234375},{"t_s":60,"rss_mib":669.40234375},{"t_s":63,"rss_mib":669.40234375},{"t_s":66,"rss_mib":669.40234375},{"t_s":69,"rss_mib":669.40234375},{"t_s":73,"rss_mib":669.40234375},{"t_s":76,"rss_mib":669.40234375},{"t_s":79,"rss_mib":669.40234375},{"t_s":82,"rss_mib":669.40234375},{"t_s":85,"rss_mib":669.40234375},{"t_s":88,"rss_mib":669.40234375},{"t_s":91,"rss_mib":669.40234375},{"t_s":94,"rss_mib":669.40234375},{"t_s":97,"rss_mib":669.40234375},{"t_s":100,"rss_mib":669.40234375},{"t_s":103,"rss_mib":669.40234375},{"t_s":106,"rss_mib":669.40234375},{"t_s":109,"rss_mib":669.40234375},{"t_s":112,"rss_mib":669.40234375},{"t_s":115,"rss_mib":669.40234375},{"t_s":118,"rss_mib":669.40234375},{"t_s":121,"rss_mib":669.40234375},{"t_s":124,"rss_mib":669.40234375},{"t_s":127,"rss_mib":669.40234375},{"t_s":130,"rss_mib":669.40234375},{"t_s":133,"rss_mib":669.40234375},{"t_s":137,"rss_mib":669.40234375},{"t_s":140,"rss_mib":669.40234375},{"t_s":143,"rss_mib":669.40234375},{"t_s":146,"rss_mib":669.40234375},{"t_s":149,"rss_mib":669.40234375},{"t_s":152,"rss_mib":669.40234375},{"t_s":155,"rss_mib":669.40234375},{"t_s":158,"rss_mib":669.40234375},{"t_s":161,"rss_mib":669.40234375},{"t_s":164,"rss_mib":669.40234375},{"t_s":167,"rss_mib":669.40234375},{"t_s":170,"rss_mib":669.40234375},{"t_s":173,"rss_mib":669.40234375},{"t_s":176,"rss_mib":669.40234375},{"t_s":179,"rss_mib":669.40234375},{"t_s":182,"rss_mib":669.40234375},{"t_s":185,"rss_mib":669.40234375},{"t_s":188,"rss_mib":669.40234375},{"t_s":191,"rss_mib":669.40234375},{"t_s":194,"rss_mib":669.40234375},{"t_s":197,"rss_mib":669.40234375},{"t_s":200,"rss_mib":669.40234375},{"t_s":204,"rss_mib":669.40234375},{"t_s":207,"rss_mib":669.40234375},{"t_s":210,"rss_mib":669.40234375},{"t_s":213,"rss_mib":669.40234375},{"t_s":216,"rss_mib":669.40234375},{"t_s":219,"rss_mib":669.40234375},{"t_s":222,"rss_mib":669.40234375},{"t_s":225,"rss_mib":669.40234375},{"t_s":228,"rss_mib":669.40234375},{"t_s":231,"rss_mib":669.40234375},{"t_s":234,"rss_mib":669.40234375},{"t_s":237,"rss_mib":669.40234375},{"t_s":240,"rss_mib":669.40234375},{"t_s":243,"rss_mib":669.40234375},{"t_s":246,"rss_mib":669.40234375},{"t_s":249,"rss_mib":669.40234375},{"t_s":252,"rss_mib":669.40234375},{"t_s":255,"rss_mib":669.40234375},{"t_s":258,"rss_mib":669.40234375},{"t_s":261,"rss_mib":669.40234375},{"t_s":264,"rss_mib":669.40234375},{"t_s":268,"rss_mib":669.40234375},{"t_s":270,"rss_mib":669.40234375},{"t_s":274,"rss_mib":669.40234375},{"t_s":277,"rss_mib":669.40234375},{"t_s":280,"rss_mib":669.40234375},{"t_s":283,"rss_mib":669.40234375},{"t_s":286,"rss_mib":669.40234375},{"t_s":289,"rss_mib":669.40234375},{"t_s":292,"rss_mib":669.40234375},{"t_s":295,"rss_mib":669.40234375},{"t_s":298,"rss_mib":669.40234375},{"t_s":301,"rss_mib":669.40234375},{"t_s":304,"rss_mib":669.40234375},{"t_s":307,"rss_mib":669.40234375},{"t_s":310,"rss_mib":669.40234375},{"t_s":313,"rss_mib":669.40234375},{"t_s":316,"rss_mib":669.40234375},{"t_s":319,"rss_mib":669.40234375},{"t_s":322,"rss_mib":669.40234375},{"t_s":325,"rss_mib":669.40234375},{"t_s":328,"rss_mib":669.40234375},{"t_s":331,"rss_mib":669.40234375},{"t_s":334,"rss_mib":669.40234375},{"t_s":337,"rss_mib":669.40234375},{"t_s":341,"rss_mib":669.40234375},{"t_s":344,"rss_mib":669.40234375},{"t_s":347,"rss_mib":669.40234375},{"t_s":350,"rss_mib":669.40234375},{"t_s":353,"rss_mib":669.40234375},{"t_s":356,"rss_mib":669.40234375},{"t_s":359,"rss_mib":669.40234375},{"t_s":362,"rss_mib":669.40234375},{"t_s":365,"rss_mib":669.40234375}],"idle_rss_series":[{"t_s":0,"rss_mib":591.7890625},{"t_s":0,"rss_mib":591.7890625},{"t_s":0,"rss_mib":591.7890625},{"t_s":1,"rss_mib":591.7890625},{"t_s":1,"rss_mib":591.7890625},{"t_s":2,"rss_mib":591.7890625},{"t_s":2,"rss_mib":591.7890625},{"t_s":3,"rss_mib":591.7890625},{"t_s":3,"rss_mib":591.7890625},{"t_s":4,"rss_mib":591.7890625},{"t_s":5,"rss_mib":591.7890625},{"t_s":5,"rss_mib":591.7890625},{"t_s":5,"rss_mib":591.7890625},{"t_s":6,"rss_mib":591.7890625},{"t_s":6,"rss_mib":591.7890625},{"t_s":7,"rss_mib":591.7890625},{"t_s":7,"rss_mib":591.7890625},{"t_s":8,"rss_mib":591.7890625},{"t_s":8,"rss_mib":591.7890625},{"t_s":9,"rss_mib":591.7890625},{"t_s":10,"rss_mib":591.7890625},{"t_s":10,"rss_mib":591.7890625},{"t_s":10,"rss_mib":591.7890625},{"t_s":11,"rss_mib":591.7890625},{"t_s":11,"rss_mib":591.7890625},{"t_s":12,"rss_mib":591.7890625},{"t_s":12,"rss_mib":591.7890625},{"t_s":13,"rss_mib":591.7890625},{"t_s":13,"rss_mib":591.7890625},{"t_s":14,"rss_mib":591.7890625},{"t_s":15,"rss_mib":591.7890625},{"t_s":15,"rss_mib":591.7890625},{"t_s":15,"rss_mib":591.7890625},{"t_s":16,"rss_mib":591.7890625},{"t_s":16,"rss_mib":591.7890625},{"t_s":17,"rss_mib":591.7890625},{"t_s":17,"rss_mib":591.7890625},{"t_s":18,"rss_mib":591.7890625},{"t_s":18,"rss_mib":591.7890625},{"t_s":19,"rss_mib":591.7890625},{"t_s":20,"rss_mib":591.7890625},{"t_s":20,"rss_mib":591.7890625},{"t_s":20,"rss_mib":591.7890625},{"t_s":21,"rss_mib":591.7890625},{"t_s":21,"rss_mib":591.7890625},{"t_s":22,"rss_mib":591.7890625},{"t_s":22,"rss_mib":591.7890625},{"t_s":23,"rss_mib":591.7890625},{"t_s":23,"rss_mib":591.7890625},{"t_s":24,"rss_mib":591.7890625},{"t_s":25,"rss_mib":591.7890625},{"t_s":25,"rss_mib":591.7890625},{"t_s":25,"rss_mib":591.7890625},{"t_s":26,"rss_mib":591.7890625},{"t_s":26,"rss_mib":591.7890625},{"t_s":27,"rss_mib":591.7890625},{"t_s":27,"rss_mib":591.7890625},{"t_s":28,"rss_mib":591.7890625},{"t_s":28,"rss_mib":591.7890625},{"t_s":29,"rss_mib":591.7890625},{"t_s":30,"rss_mib":591.7890625},{"t_s":30,"rss_mib":591.7890625},{"t_s":30,"rss_mib":591.7890625},{"t_s":31,"rss_mib":591.7890625},{"t_s":31,"rss_mib":591.7890625},{"t_s":32,"rss_mib":591.7890625},{"t_s":32,"rss_mib":591.7890625},{"t_s":33,"rss_mib":591.7890625},{"t_s":33,"rss_mib":591.7890625},{"t_s":34,"rss_mib":591.7890625},{"t_s":35,"rss_mib":591.7890625},{"t_s":35,"rss_mib":591.7890625},{"t_s":35,"rss_mib":591.7890625},{"t_s":36,"rss_mib":591.7890625},{"t_s":36,"rss_mib":591.7890625},{"t_s":37,"rss_mib":591.7890625},{"t_s":37,"rss_mib":591.7890625},{"t_s":38,"rss_mib":591.7890625},{"t_s":38,"rss_mib":591.7890625},{"t_s":39,"rss_mib":591.7890625},{"t_s":40,"rss_mib":591.7890625},{"t_s":40,"rss_mib":591.7890625},{"t_s":40,"rss_mib":591.7890625},{"t_s":41,"rss_mib":591.7890625},{"t_s":41,"rss_mib":591.7890625},{"t_s":42,"rss_mib":591.7890625},{"t_s":42,"rss_mib":591.7890625},{"t_s":43,"rss_mib":591.7890625},{"t_s":43,"rss_mib":591.7890625},{"t_s":44,"rss_mib":591.7890625},{"t_s":45,"rss_mib":591.7890625},{"t_s":45,"rss_mib":591.7890625},{"t_s":45,"rss_mib":591.7890625},{"t_s":46,"rss_mib":591.7890625},{"t_s":46,"rss_mib":591.7890625},{"t_s":47,"rss_mib":591.7890625},{"t_s":47,"rss_mib":591.7890625},{"t_s":48,"rss_mib":591.7890625},{"t_s":48,"rss_mib":591.7890625},{"t_s":49,"rss_mib":591.7890625},{"t_s":50,"rss_mib":591.7890625},{"t_s":50,"rss_mib":591.7890625},{"t_s":50,"rss_mib":591.7890625},{"t_s":51,"rss_mib":591.7890625},{"t_s":51,"rss_mib":591.7890625},{"t_s":52,"rss_mib":591.7890625},{"t_s":52,"rss_mib":591.7890625},{"t_s":53,"rss_mib":591.7890625},{"t_s":54,"rss_mib":591.7890625},{"t_s":54,"rss_mib":591.7890625},{"t_s":55,"rss_mib":591.7890625},{"t_s":55,"rss_mib":591.7890625},{"t_s":55,"rss_mib":591.7890625},{"t_s":56,"rss_mib":591.7890625},{"t_s":56,"rss_mib":591.7890625},{"t_s":57,"rss_mib":591.7890625},{"t_s":57,"rss_mib":591.7890625},{"t_s":58,"rss_mib":591.7890625},{"t_s":59,"rss_mib":591.7890625},{"t_s":59,"rss_mib":591.7890625},{"t_s":59,"rss_mib":591.7890625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.017513885,"cost":6.087797019,"memory":440.822326975,"streaming":0.000002234,"streams_sustained":0.000001853,"throughput":641.736236358},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":11554,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":12280,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":12308,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":11692.307692307691,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":85.52631578947368,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2497786700112885,"certified":true,"suppressed":false},"cost_window_ok":{"value":520,"certified":true,"suppressed":false},"cost_window_rps":{"value":86,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.06346153846153846,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":87,"certified":true,"suppressed":false},"concurrency":2,"p99_us":23671,"first_disqualified_conc":8,"lower_bound":false,"samples":524},{"bound_ms":100,"rps":{"value":87,"certified":true,"suppressed":false},"concurrency":2,"p99_us":23671,"first_disqualified_conc":16,"lower_bound":false,"samples":524},{"bound_ms":null,"rps":{"value":88,"certified":true,"suppressed":false},"concurrency":32,"p99_us":556619,"first_disqualified_conc":8192,"lower_bound":false,"samples":529}],"sweep":[{"conc":1,"ok":520,"rps":86,"p99_us":12207,"fail":0},{"conc":1,"ok":514,"rps":85,"p99_us":12402,"fail":0},{"conc":1,"ok":514,"rps":85,"p99_us":12379,"fail":0},{"conc":2,"ok":528,"rps":87,"p99_us":23688,"fail":0},{"conc":2,"ok":526,"rps":87,"p99_us":23576,"fail":0},{"conc":2,"ok":524,"rps":87,"p99_us":23671,"fail":0},{"conc":4,"ok":524,"rps":87,"p99_us":48317,"fail":0},{"conc":4,"ok":523,"rps":87,"p99_us":49127,"fail":0},{"conc":4,"ok":521,"rps":86,"p99_us":48490,"fail":0},{"conc":8,"ok":520,"rps":86,"p99_us":99080,"fail":0},{"conc":8,"ok":523,"rps":87,"p99_us":99079,"fail":0},{"conc":8,"ok":525,"rps":87,"p99_us":97938,"fail":0},{"conc":16,"ok":524,"rps":87,"p99_us":199142,"fail":0},{"conc":16,"ok":525,"rps":87,"p99_us":201814,"fail":0},{"conc":16,"ok":525,"rps":87,"p99_us":198898,"fail":0},{"conc":32,"ok":524,"rps":87,"p99_us":554831,"fail":0},{"conc":32,"ok":525,"rps":87,"p99_us":531731,"fail":0},{"conc":32,"ok":529,"rps":88,"p99_us":556619,"fail":0},{"conc":64,"ok":480,"rps":79,"p99_us":2002428,"fail":0},{"conc":64,"ok":518,"rps":86,"p99_us":1982369,"fail":0},{"conc":64,"ok":520,"rps":86,"p99_us":1945822,"fail":0},{"conc":128,"ok":501,"rps":83,"p99_us":5400748,"fail":0},{"conc":128,"ok":497,"rps":82,"p99_us":2010383,"fail":0},{"conc":128,"ok":504,"rps":83,"p99_us":5459427,"fail":0},{"conc":256,"ok":507,"rps":84,"p99_us":3294121,"fail":0},{"conc":256,"ok":479,"rps":79,"p99_us":5552899,"fail":0},{"conc":256,"ok":511,"rps":85,"p99_us":2983242,"fail":0},{"conc":512,"ok":429,"rps":71,"p99_us":5565454,"fail":0},{"conc":512,"ok":325,"rps":54,"p99_us":4202437,"fail":0},{"conc":512,"ok":499,"rps":83,"p99_us":5523521,"fail":0},{"conc":1024,"ok":509,"rps":84,"p99_us":5459124,"fail":0},{"conc":1024,"ok":355,"rps":59,"p99_us":4575209,"fail":0},{"conc":1024,"ok":505,"rps":84,"p99_us":5481110,"fail":0},{"conc":2048,"ok":356,"rps":59,"p99_us":4893331,"fail":0},{"conc":2048,"ok":370,"rps":61,"p99_us":4400961,"fail":0},{"conc":2048,"ok":468,"rps":77,"p99_us":5540251,"fail":0},{"conc":4096,"ok":504,"rps":83,"p99_us":5498035,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":3387},{"conc":8192,"ok":361,"rps":60,"p99_us":4700198,"fail":4267},{"conc":8192,"ok":16,"rps":2,"p99_us":5966017,"fail":658}],"rungs":[{"conc":1,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":12379,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":23671,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":48490,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":99079,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":199142,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":87,"certified":true,"suppressed":false},"p99_us":554831,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":86,"certified":true,"suppressed":false},"p99_us":1982369,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":5400748,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":84,"certified":true,"suppressed":false},"p99_us":3294121,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":5523521,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":84,"certified":true,"suppressed":false},"p99_us":5459124,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":4893331,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":83,"certified":true,"suppressed":false},"p99_us":5498035,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=8192 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"openai-responses>openai-responses is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 518 successful gateway round trip(s) and 265921 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":518,"direct_c1_samples":265921},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.46484375,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":627.4453125,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":629.47265625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":629.47265625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":629.47265625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":3.0355931092621313,"certified":true,"suppressed":false},"load_s":{"value":300,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.46484375},{"t_s":2,"rss_mib":624.453125},{"t_s":2,"rss_mib":624.453125},{"t_s":5,"rss_mib":624.9453125},{"t_s":5,"rss_mib":625},{"t_s":8,"rss_mib":625.140625},{"t_s":8,"rss_mib":625.14453125},{"t_s":11,"rss_mib":625.23828125},{"t_s":11,"rss_mib":625.23828125},{"t_s":13,"rss_mib":625.24609375},{"t_s":15,"rss_mib":625.24609375},{"t_s":17,"rss_mib":625.27734375},{"t_s":17,"rss_mib":625.27734375},{"t_s":20,"rss_mib":625.296875},{"t_s":20,"rss_mib":625.296875},{"t_s":23,"rss_mib":625.3046875},{"t_s":24,"rss_mib":625.3046875},{"t_s":26,"rss_mib":625.3125},{"t_s":26,"rss_mib":625.3125},{"t_s":27,"rss_mib":625.328125},{"t_s":30,"rss_mib":625.328125},{"t_s":31,"rss_mib":625.33203125},{"t_s":33,"rss_mib":625.33203125},{"t_s":35,"rss_mib":625.33984375},{"t_s":36,"rss_mib":625.33984375},{"t_s":38,"rss_mib":625.3515625},{"t_s":39,"rss_mib":625.3515625},{"t_s":41,"rss_mib":625.3515625},{"t_s":44,"rss_mib":627.3828125},{"t_s":45,"rss_mib":627.3828125},{"t_s":48,"rss_mib":627.3828125},{"t_s":51,"rss_mib":627.3828125},{"t_s":54,"rss_mib":627.3828125},{"t_s":55,"rss_mib":627.38671875},{"t_s":57,"rss_mib":627.390625},{"t_s":58,"rss_mib":627.3984375},{"t_s":60,"rss_mib":627.3984375},{"t_s":63,"rss_mib":627.3984375},{"t_s":66,"rss_mib":627.3984375},{"t_s":69,"rss_mib":627.3984375},{"t_s":72,"rss_mib":627.40625},{"t_s":72,"rss_mib":627.40625},{"t_s":74,"rss_mib":627.421875},{"t_s":75,"rss_mib":627.421875},{"t_s":78,"rss_mib":627.421875},{"t_s":78,"rss_mib":627.42578125},{"t_s":81,"rss_mib":627.42578125},{"t_s":84,"rss_mib":627.42578125},{"t_s":87,"rss_mib":627.42578125},{"t_s":90,"rss_mib":627.42578125},{"t_s":93,"rss_mib":627.42578125},{"t_s":96,"rss_mib":627.42578125},{"t_s":98,"rss_mib":627.4296875},{"t_s":99,"rss_mib":627.4296875},{"t_s":102,"rss_mib":627.4296875},{"t_s":105,"rss_mib":627.4296875},{"t_s":108,"rss_mib":627.4296875},{"t_s":111,"rss_mib":627.4296875},{"t_s":114,"rss_mib":627.4296875},{"t_s":117,"rss_mib":627.4296875},{"t_s":120,"rss_mib":627.4296875},{"t_s":123,"rss_mib":627.4296875},{"t_s":126,"rss_mib":627.4296875},{"t_s":129,"rss_mib":627.4296875},{"t_s":132,"rss_mib":627.4296875},{"t_s":135,"rss_mib":627.4296875},{"t_s":138,"rss_mib":627.4296875},{"t_s":141,"rss_mib":627.4296875},{"t_s":144,"rss_mib":627.4296875},{"t_s":147,"rss_mib":627.4296875},{"t_s":150,"rss_mib":627.4296875},{"t_s":153,"rss_mib":627.4296875},{"t_s":156,"rss_mib":627.4296875},{"t_s":159,"rss_mib":627.4296875},{"t_s":162,"rss_mib":627.4296875},{"t_s":165,"rss_mib":627.4296875},{"t_s":167,"rss_mib":627.4375},{"t_s":168,"rss_mib":627.4375},{"t_s":168,"rss_mib":627.44140625},{"t_s":171,"rss_mib":627.44140625},{"t_s":174,"rss_mib":627.44140625},{"t_s":177,"rss_mib":627.44140625},{"t_s":180,"rss_mib":627.44140625},{"t_s":183,"rss_mib":627.44140625},{"t_s":183,"rss_mib":627.4453125},{"t_s":186,"rss_mib":627.4453125},{"t_s":189,"rss_mib":627.4453125},{"t_s":192,"rss_mib":627.4453125},{"t_s":195,"rss_mib":627.4453125},{"t_s":198,"rss_mib":627.4453125},{"t_s":201,"rss_mib":627.4453125},{"t_s":204,"rss_mib":627.4453125},{"t_s":207,"rss_mib":627.4453125},{"t_s":210,"rss_mib":627.4453125},{"t_s":213,"rss_mib":627.4453125},{"t_s":216,"rss_mib":627.4453125},{"t_s":219,"rss_mib":627.4453125},{"t_s":222,"rss_mib":627.4453125},{"t_s":225,"rss_mib":627.4453125},{"t_s":228,"rss_mib":627.4453125},{"t_s":231,"rss_mib":627.4453125},{"t_s":234,"rss_mib":627.4453125},{"t_s":237,"rss_mib":627.4453125},{"t_s":240,"rss_mib":627.4453125},{"t_s":243,"rss_mib":627.4453125},{"t_s":246,"rss_mib":627.4453125},{"t_s":249,"rss_mib":627.4453125},{"t_s":252,"rss_mib":627.4453125},{"t_s":255,"rss_mib":627.4453125},{"t_s":258,"rss_mib":627.4453125},{"t_s":261,"rss_mib":627.4453125},{"t_s":264,"rss_mib":627.4453125},{"t_s":267,"rss_mib":627.4453125},{"t_s":270,"rss_mib":627.4453125},{"t_s":271,"rss_mib":629.47265625},{"t_s":273,"rss_mib":629.47265625},{"t_s":276,"rss_mib":629.47265625},{"t_s":279,"rss_mib":629.47265625},{"t_s":282,"rss_mib":629.47265625},{"t_s":285,"rss_mib":629.47265625},{"t_s":288,"rss_mib":629.47265625},{"t_s":291,"rss_mib":629.47265625},{"t_s":294,"rss_mib":629.47265625},{"t_s":297,"rss_mib":629.47265625},{"t_s":300,"rss_mib":629.47265625},{"t_s":303,"rss_mib":629.47265625},{"t_s":306,"rss_mib":629.47265625},{"t_s":309,"rss_mib":629.47265625},{"t_s":312,"rss_mib":629.47265625},{"t_s":315,"rss_mib":629.47265625},{"t_s":318,"rss_mib":629.47265625},{"t_s":321,"rss_mib":629.47265625},{"t_s":324,"rss_mib":629.47265625},{"t_s":327,"rss_mib":629.47265625},{"t_s":330,"rss_mib":629.47265625},{"t_s":333,"rss_mib":629.47265625},{"t_s":336,"rss_mib":629.47265625},{"t_s":339,"rss_mib":629.47265625},{"t_s":342,"rss_mib":629.47265625},{"t_s":345,"rss_mib":629.47265625},{"t_s":348,"rss_mib":629.47265625},{"t_s":351,"rss_mib":629.47265625},{"t_s":354,"rss_mib":629.47265625},{"t_s":357,"rss_mib":629.47265625},{"t_s":360,"rss_mib":629.47265625}],"idle_rss_series":[{"t_s":0,"rss_mib":592.46484375},{"t_s":0,"rss_mib":592.46484375},{"t_s":0,"rss_mib":592.46484375},{"t_s":1,"rss_mib":592.46484375},{"t_s":1,"rss_mib":592.46484375},{"t_s":2,"rss_mib":592.46484375},{"t_s":2,"rss_mib":592.46484375},{"t_s":3,"rss_mib":592.46484375},{"t_s":3,"rss_mib":592.46484375},{"t_s":4,"rss_mib":592.46484375},{"t_s":5,"rss_mib":592.46484375},{"t_s":5,"rss_mib":592.46484375},{"t_s":5,"rss_mib":592.46484375},{"t_s":6,"rss_mib":592.46484375},{"t_s":6,"rss_mib":592.46484375},{"t_s":7,"rss_mib":592.46484375},{"t_s":7,"rss_mib":592.46484375},{"t_s":8,"rss_mib":592.46484375},{"t_s":8,"rss_mib":592.46484375},{"t_s":9,"rss_mib":592.46484375},{"t_s":10,"rss_mib":592.46484375},{"t_s":10,"rss_mib":592.46484375},{"t_s":10,"rss_mib":592.46484375},{"t_s":11,"rss_mib":592.46484375},{"t_s":11,"rss_mib":592.46484375},{"t_s":12,"rss_mib":592.46484375},{"t_s":12,"rss_mib":592.46484375},{"t_s":13,"rss_mib":592.46484375},{"t_s":13,"rss_mib":592.46484375},{"t_s":14,"rss_mib":592.46484375},{"t_s":15,"rss_mib":592.46484375},{"t_s":15,"rss_mib":592.46484375},{"t_s":15,"rss_mib":592.46484375},{"t_s":16,"rss_mib":592.46484375},{"t_s":16,"rss_mib":592.46484375},{"t_s":17,"rss_mib":592.46484375},{"t_s":17,"rss_mib":592.46484375},{"t_s":18,"rss_mib":592.46484375},{"t_s":18,"rss_mib":592.46484375},{"t_s":19,"rss_mib":592.46484375},{"t_s":20,"rss_mib":592.46484375},{"t_s":20,"rss_mib":592.46484375},{"t_s":20,"rss_mib":592.46484375},{"t_s":21,"rss_mib":592.46484375},{"t_s":21,"rss_mib":592.46484375},{"t_s":22,"rss_mib":592.46484375},{"t_s":22,"rss_mib":592.46484375},{"t_s":23,"rss_mib":592.46484375},{"t_s":23,"rss_mib":592.46484375},{"t_s":24,"rss_mib":592.46484375},{"t_s":25,"rss_mib":592.46484375},{"t_s":25,"rss_mib":592.46484375},{"t_s":25,"rss_mib":592.46484375},{"t_s":26,"rss_mib":592.46484375},{"t_s":26,"rss_mib":592.46484375},{"t_s":27,"rss_mib":592.46484375},{"t_s":27,"rss_mib":592.46484375},{"t_s":28,"rss_mib":592.46484375},{"t_s":28,"rss_mib":592.46484375},{"t_s":29,"rss_mib":592.46484375},{"t_s":30,"rss_mib":592.46484375},{"t_s":30,"rss_mib":592.46484375},{"t_s":30,"rss_mib":592.46484375},{"t_s":31,"rss_mib":592.46484375},{"t_s":31,"rss_mib":592.46484375},{"t_s":32,"rss_mib":592.46484375},{"t_s":32,"rss_mib":592.46484375},{"t_s":33,"rss_mib":592.46484375},{"t_s":33,"rss_mib":592.46484375},{"t_s":34,"rss_mib":592.46484375},{"t_s":35,"rss_mib":592.46484375},{"t_s":35,"rss_mib":592.46484375},{"t_s":35,"rss_mib":592.46484375},{"t_s":36,"rss_mib":592.46484375},{"t_s":36,"rss_mib":592.46484375},{"t_s":37,"rss_mib":592.46484375},{"t_s":37,"rss_mib":592.46484375},{"t_s":38,"rss_mib":592.46484375},{"t_s":38,"rss_mib":592.46484375},{"t_s":39,"rss_mib":592.46484375},{"t_s":40,"rss_mib":592.46484375},{"t_s":40,"rss_mib":592.46484375},{"t_s":40,"rss_mib":592.46484375},{"t_s":41,"rss_mib":592.46484375},{"t_s":41,"rss_mib":592.46484375},{"t_s":42,"rss_mib":592.46484375},{"t_s":42,"rss_mib":592.46484375},{"t_s":43,"rss_mib":592.46484375},{"t_s":43,"rss_mib":592.46484375},{"t_s":44,"rss_mib":592.46484375},{"t_s":45,"rss_mib":592.46484375},{"t_s":45,"rss_mib":592.46484375},{"t_s":45,"rss_mib":592.46484375},{"t_s":46,"rss_mib":592.46484375},{"t_s":46,"rss_mib":592.46484375},{"t_s":47,"rss_mib":592.46484375},{"t_s":47,"rss_mib":592.46484375},{"t_s":48,"rss_mib":592.46484375},{"t_s":48,"rss_mib":592.46484375},{"t_s":49,"rss_mib":592.46484375},{"t_s":50,"rss_mib":592.46484375},{"t_s":50,"rss_mib":592.46484375},{"t_s":50,"rss_mib":592.46484375},{"t_s":51,"rss_mib":592.46484375},{"t_s":51,"rss_mib":592.46484375},{"t_s":52,"rss_mib":592.46484375},{"t_s":52,"rss_mib":592.46484375},{"t_s":53,"rss_mib":592.46484375},{"t_s":53,"rss_mib":592.46484375},{"t_s":54,"rss_mib":592.46484375},{"t_s":55,"rss_mib":592.46484375},{"t_s":55,"rss_mib":592.46484375},{"t_s":55,"rss_mib":592.46484375},{"t_s":56,"rss_mib":592.46484375},{"t_s":56,"rss_mib":592.46484375},{"t_s":57,"rss_mib":592.46484375},{"t_s":57,"rss_mib":592.46484375},{"t_s":58,"rss_mib":592.46484375},{"t_s":58,"rss_mib":592.46484375},{"t_s":59,"rss_mib":592.46484375},{"t_s":59,"rss_mib":592.46484375}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.011457198,"cost":6.098785566,"memory":436.001350025,"streaming":0.000002152,"streams_sustained":0.000001608,"throughput":554.685631683},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w1.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}}}},"gemini":{"configurable":true,"served":true,"egress_config":null,"serve_error":"","cells":{"openai-responses":{"served":true,"reason":null,"status":"","path":"/v1/responses","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10486,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":11201,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":11229,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10628.272251308901,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":94.08866995073892,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25043632937727195,"certified":true,"suppressed":false},"cost_window_ok":{"value":573,"certified":true,"suppressed":false},"cost_window_rps":{"value":95,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.041884816753926704,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":96,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21588,"first_disqualified_conc":8,"lower_bound":false,"samples":578},{"bound_ms":100,"rps":{"value":96,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21588,"first_disqualified_conc":16,"lower_bound":false,"samples":578},{"bound_ms":null,"rps":{"value":96,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21588,"first_disqualified_conc":16384,"lower_bound":false,"samples":578}],"sweep":[{"conc":1,"ok":571,"rps":95,"p99_us":11101,"fail":0},{"conc":1,"ok":566,"rps":94,"p99_us":11212,"fail":0},{"conc":1,"ok":567,"rps":94,"p99_us":11236,"fail":0},{"conc":2,"ok":570,"rps":94,"p99_us":21841,"fail":0},{"conc":2,"ok":578,"rps":96,"p99_us":21605,"fail":0},{"conc":2,"ok":578,"rps":96,"p99_us":21588,"fail":0},{"conc":4,"ok":573,"rps":95,"p99_us":44889,"fail":0},{"conc":4,"ok":575,"rps":95,"p99_us":45153,"fail":0},{"conc":4,"ok":578,"rps":96,"p99_us":44374,"fail":0},{"conc":8,"ok":576,"rps":95,"p99_us":90051,"fail":0},{"conc":8,"ok":533,"rps":88,"p99_us":512508,"fail":0},{"conc":8,"ok":575,"rps":95,"p99_us":89331,"fail":0},{"conc":16,"ok":575,"rps":95,"p99_us":180247,"fail":0},{"conc":16,"ok":576,"rps":95,"p99_us":178773,"fail":0},{"conc":16,"ok":575,"rps":95,"p99_us":179171,"fail":0},{"conc":32,"ok":573,"rps":95,"p99_us":502262,"fail":0},{"conc":32,"ok":575,"rps":95,"p99_us":477682,"fail":0},{"conc":32,"ok":575,"rps":95,"p99_us":479218,"fail":0},{"conc":64,"ok":569,"rps":94,"p99_us":1621782,"fail":0},{"conc":64,"ok":576,"rps":95,"p99_us":712981,"fail":0},{"conc":64,"ok":529,"rps":88,"p99_us":1099216,"fail":0},{"conc":128,"ok":542,"rps":90,"p99_us":5424302,"fail":0},{"conc":128,"ok":518,"rps":86,"p99_us":5310548,"fail":0},{"conc":128,"ok":546,"rps":90,"p99_us":1437943,"fail":0},{"conc":256,"ok":493,"rps":82,"p99_us":5474924,"fail":0},{"conc":256,"ok":538,"rps":89,"p99_us":5415283,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":2674520,"fail":0},{"conc":512,"ok":540,"rps":89,"p99_us":5515647,"fail":0},{"conc":512,"ok":400,"rps":66,"p99_us":4911814,"fail":0},{"conc":512,"ok":373,"rps":61,"p99_us":4888959,"fail":0},{"conc":1024,"ok":502,"rps":83,"p99_us":5482042,"fail":0},{"conc":1024,"ok":349,"rps":58,"p99_us":5136989,"fail":0},{"conc":1024,"ok":363,"rps":60,"p99_us":4894812,"fail":0},{"conc":2048,"ok":369,"rps":61,"p99_us":5523557,"fail":0},{"conc":2048,"ok":539,"rps":89,"p99_us":5564018,"fail":0},{"conc":2048,"ok":376,"rps":62,"p99_us":5170707,"fail":0},{"conc":4096,"ok":367,"rps":61,"p99_us":5208754,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":444},{"conc":8192,"ok":33,"rps":5,"p99_us":5950704,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":1739},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":914},{"conc":16384,"ok":516,"rps":85,"p99_us":4118402,"fail":4010},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":1379}],"rungs":[{"conc":1,"rps":{"value":94,"certified":true,"suppressed":false},"p99_us":11212,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":96,"certified":true,"suppressed":false},"p99_us":21605,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":95,"certified":true,"suppressed":false},"p99_us":44889,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":95,"certified":true,"suppressed":false},"p99_us":90051,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":95,"certified":true,"suppressed":false},"p99_us":179171,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":95,"certified":true,"suppressed":false},"p99_us":479218,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":94,"certified":true,"suppressed":false},"p99_us":1099216,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":90,"certified":true,"suppressed":false},"p99_us":5310548,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5415283,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":66,"certified":true,"suppressed":false},"p99_us":4911814,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":60,"certified":true,"suppressed":false},"p99_us":5136989,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":62,"certified":true,"suppressed":false},"p99_us":5523557,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":5208754,"windows":3,"clean_windows":1},{"conc":8192,"rps":{"value":5,"certified":true,"suppressed":false},"p99_us":5950704,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 570 successful gateway round trip(s) and 265239 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":570,"direct_c1_samples":265239},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":592.390625,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":669.41015625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":669.4140625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":669.4140625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":669.4140625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":304,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":592.390625},{"t_s":2,"rss_mib":669.41015625},{"t_s":2,"rss_mib":669.41015625},{"t_s":6,"rss_mib":669.41015625},{"t_s":9,"rss_mib":669.41015625},{"t_s":12,"rss_mib":669.41015625},{"t_s":15,"rss_mib":669.41015625},{"t_s":18,"rss_mib":669.41015625},{"t_s":21,"rss_mib":669.41015625},{"t_s":24,"rss_mib":669.41015625},{"t_s":27,"rss_mib":669.41015625},{"t_s":30,"rss_mib":669.41015625},{"t_s":33,"rss_mib":669.41015625},{"t_s":36,"rss_mib":669.41015625},{"t_s":39,"rss_mib":669.41015625},{"t_s":42,"rss_mib":669.41015625},{"t_s":45,"rss_mib":669.41015625},{"t_s":48,"rss_mib":669.41015625},{"t_s":51,"rss_mib":669.41015625},{"t_s":54,"rss_mib":669.41015625},{"t_s":57,"rss_mib":669.41015625},{"t_s":60,"rss_mib":669.41015625},{"t_s":63,"rss_mib":669.41015625},{"t_s":66,"rss_mib":669.41015625},{"t_s":69,"rss_mib":669.41015625},{"t_s":72,"rss_mib":669.41015625},{"t_s":76,"rss_mib":669.41015625},{"t_s":78,"rss_mib":669.41015625},{"t_s":82,"rss_mib":669.41015625},{"t_s":85,"rss_mib":669.41015625},{"t_s":88,"rss_mib":669.41015625},{"t_s":91,"rss_mib":669.41015625},{"t_s":94,"rss_mib":669.41015625},{"t_s":97,"rss_mib":669.41015625},{"t_s":100,"rss_mib":669.41015625},{"t_s":103,"rss_mib":669.41015625},{"t_s":106,"rss_mib":669.41015625},{"t_s":109,"rss_mib":669.41015625},{"t_s":112,"rss_mib":669.41015625},{"t_s":115,"rss_mib":669.41015625},{"t_s":118,"rss_mib":669.41015625},{"t_s":121,"rss_mib":669.41015625},{"t_s":124,"rss_mib":669.41015625},{"t_s":127,"rss_mib":669.41015625},{"t_s":130,"rss_mib":669.41015625},{"t_s":133,"rss_mib":669.41015625},{"t_s":136,"rss_mib":669.41015625},{"t_s":139,"rss_mib":669.41015625},{"t_s":142,"rss_mib":669.41015625},{"t_s":145,"rss_mib":669.41015625},{"t_s":148,"rss_mib":669.41015625},{"t_s":152,"rss_mib":669.41015625},{"t_s":155,"rss_mib":669.41015625},{"t_s":158,"rss_mib":669.41015625},{"t_s":161,"rss_mib":669.41015625},{"t_s":164,"rss_mib":669.41015625},{"t_s":167,"rss_mib":669.41015625},{"t_s":170,"rss_mib":669.41015625},{"t_s":173,"rss_mib":669.41015625},{"t_s":176,"rss_mib":669.41015625},{"t_s":179,"rss_mib":669.41015625},{"t_s":182,"rss_mib":669.41015625},{"t_s":185,"rss_mib":669.41015625},{"t_s":188,"rss_mib":669.41015625},{"t_s":191,"rss_mib":669.41015625},{"t_s":194,"rss_mib":669.41015625},{"t_s":197,"rss_mib":669.41015625},{"t_s":200,"rss_mib":669.41015625},{"t_s":203,"rss_mib":669.41015625},{"t_s":206,"rss_mib":669.41015625},{"t_s":209,"rss_mib":669.41015625},{"t_s":212,"rss_mib":669.41015625},{"t_s":215,"rss_mib":669.41015625},{"t_s":218,"rss_mib":669.41015625},{"t_s":221,"rss_mib":669.41015625},{"t_s":224,"rss_mib":669.41015625},{"t_s":228,"rss_mib":669.41015625},{"t_s":231,"rss_mib":669.41015625},{"t_s":234,"rss_mib":669.41015625},{"t_s":237,"rss_mib":669.41015625},{"t_s":239,"rss_mib":669.4140625},{"t_s":240,"rss_mib":669.4140625},{"t_s":243,"rss_mib":669.4140625},{"t_s":246,"rss_mib":669.4140625},{"t_s":249,"rss_mib":669.4140625},{"t_s":252,"rss_mib":669.4140625},{"t_s":255,"rss_mib":669.4140625},{"t_s":258,"rss_mib":669.4140625},{"t_s":261,"rss_mib":669.4140625},{"t_s":264,"rss_mib":669.4140625},{"t_s":267,"rss_mib":669.4140625},{"t_s":270,"rss_mib":669.4140625},{"t_s":273,"rss_mib":669.4140625},{"t_s":276,"rss_mib":669.4140625},{"t_s":279,"rss_mib":669.4140625},{"t_s":282,"rss_mib":669.4140625},{"t_s":285,"rss_mib":669.4140625},{"t_s":288,"rss_mib":669.4140625},{"t_s":291,"rss_mib":669.4140625},{"t_s":294,"rss_mib":669.4140625},{"t_s":298,"rss_mib":669.4140625},{"t_s":300,"rss_mib":669.4140625},{"t_s":304,"rss_mib":669.4140625},{"t_s":307,"rss_mib":669.4140625},{"t_s":310,"rss_mib":669.4140625},{"t_s":313,"rss_mib":669.4140625},{"t_s":316,"rss_mib":669.4140625},{"t_s":319,"rss_mib":669.4140625},{"t_s":322,"rss_mib":669.4140625},{"t_s":325,"rss_mib":669.4140625},{"t_s":328,"rss_mib":669.4140625},{"t_s":331,"rss_mib":669.4140625},{"t_s":334,"rss_mib":669.4140625},{"t_s":337,"rss_mib":669.4140625},{"t_s":340,"rss_mib":669.4140625},{"t_s":343,"rss_mib":669.4140625},{"t_s":346,"rss_mib":669.4140625},{"t_s":349,"rss_mib":669.4140625},{"t_s":352,"rss_mib":669.4140625},{"t_s":355,"rss_mib":669.4140625},{"t_s":358,"rss_mib":669.4140625},{"t_s":361,"rss_mib":669.4140625},{"t_s":364,"rss_mib":669.4140625}],"idle_rss_series":[{"t_s":0,"rss_mib":592.390625},{"t_s":0,"rss_mib":592.390625},{"t_s":0,"rss_mib":592.390625},{"t_s":1,"rss_mib":592.390625},{"t_s":1,"rss_mib":592.390625},{"t_s":2,"rss_mib":592.390625},{"t_s":2,"rss_mib":592.390625},{"t_s":3,"rss_mib":592.390625},{"t_s":3,"rss_mib":592.390625},{"t_s":4,"rss_mib":592.390625},{"t_s":5,"rss_mib":592.390625},{"t_s":5,"rss_mib":592.390625},{"t_s":5,"rss_mib":592.390625},{"t_s":6,"rss_mib":592.390625},{"t_s":6,"rss_mib":592.390625},{"t_s":7,"rss_mib":592.390625},{"t_s":7,"rss_mib":592.390625},{"t_s":8,"rss_mib":592.390625},{"t_s":8,"rss_mib":592.390625},{"t_s":9,"rss_mib":592.390625},{"t_s":10,"rss_mib":592.390625},{"t_s":10,"rss_mib":592.390625},{"t_s":10,"rss_mib":592.390625},{"t_s":11,"rss_mib":592.390625},{"t_s":11,"rss_mib":592.390625},{"t_s":12,"rss_mib":592.390625},{"t_s":12,"rss_mib":592.390625},{"t_s":13,"rss_mib":592.390625},{"t_s":13,"rss_mib":592.390625},{"t_s":14,"rss_mib":592.390625},{"t_s":15,"rss_mib":592.390625},{"t_s":15,"rss_mib":592.390625},{"t_s":15,"rss_mib":592.390625},{"t_s":16,"rss_mib":592.390625},{"t_s":16,"rss_mib":592.390625},{"t_s":17,"rss_mib":592.390625},{"t_s":17,"rss_mib":592.390625},{"t_s":18,"rss_mib":592.390625},{"t_s":18,"rss_mib":592.390625},{"t_s":19,"rss_mib":592.390625},{"t_s":20,"rss_mib":592.390625},{"t_s":20,"rss_mib":592.390625},{"t_s":20,"rss_mib":592.390625},{"t_s":21,"rss_mib":592.390625},{"t_s":21,"rss_mib":592.390625},{"t_s":22,"rss_mib":592.390625},{"t_s":22,"rss_mib":592.390625},{"t_s":23,"rss_mib":592.390625},{"t_s":23,"rss_mib":592.390625},{"t_s":24,"rss_mib":592.390625},{"t_s":25,"rss_mib":592.390625},{"t_s":25,"rss_mib":592.390625},{"t_s":25,"rss_mib":592.390625},{"t_s":26,"rss_mib":592.390625},{"t_s":26,"rss_mib":592.390625},{"t_s":27,"rss_mib":592.390625},{"t_s":27,"rss_mib":592.390625},{"t_s":28,"rss_mib":592.390625},{"t_s":28,"rss_mib":592.390625},{"t_s":29,"rss_mib":592.390625},{"t_s":30,"rss_mib":592.390625},{"t_s":30,"rss_mib":592.390625},{"t_s":30,"rss_mib":592.390625},{"t_s":31,"rss_mib":592.390625},{"t_s":31,"rss_mib":592.390625},{"t_s":32,"rss_mib":592.390625},{"t_s":32,"rss_mib":592.390625},{"t_s":33,"rss_mib":592.390625},{"t_s":33,"rss_mib":592.390625},{"t_s":34,"rss_mib":592.390625},{"t_s":35,"rss_mib":592.390625},{"t_s":35,"rss_mib":592.390625},{"t_s":35,"rss_mib":592.390625},{"t_s":36,"rss_mib":592.390625},{"t_s":36,"rss_mib":592.390625},{"t_s":37,"rss_mib":592.390625},{"t_s":37,"rss_mib":592.390625},{"t_s":38,"rss_mib":592.390625},{"t_s":38,"rss_mib":592.390625},{"t_s":39,"rss_mib":592.390625},{"t_s":40,"rss_mib":592.390625},{"t_s":40,"rss_mib":592.390625},{"t_s":40,"rss_mib":592.390625},{"t_s":41,"rss_mib":592.390625},{"t_s":41,"rss_mib":592.390625},{"t_s":42,"rss_mib":592.390625},{"t_s":42,"rss_mib":592.390625},{"t_s":43,"rss_mib":592.390625},{"t_s":43,"rss_mib":592.390625},{"t_s":44,"rss_mib":592.390625},{"t_s":45,"rss_mib":592.390625},{"t_s":45,"rss_mib":592.390625},{"t_s":45,"rss_mib":592.390625},{"t_s":46,"rss_mib":592.390625},{"t_s":46,"rss_mib":592.390625},{"t_s":47,"rss_mib":592.390625},{"t_s":47,"rss_mib":592.390625},{"t_s":48,"rss_mib":592.390625},{"t_s":49,"rss_mib":592.390625},{"t_s":49,"rss_mib":592.390625},{"t_s":50,"rss_mib":592.390625},{"t_s":50,"rss_mib":592.390625},{"t_s":50,"rss_mib":592.390625},{"t_s":51,"rss_mib":592.390625},{"t_s":51,"rss_mib":592.390625},{"t_s":52,"rss_mib":592.390625},{"t_s":52,"rss_mib":592.390625},{"t_s":53,"rss_mib":592.390625},{"t_s":54,"rss_mib":592.390625},{"t_s":54,"rss_mib":592.390625},{"t_s":55,"rss_mib":592.390625},{"t_s":55,"rss_mib":592.390625},{"t_s":55,"rss_mib":592.390625},{"t_s":56,"rss_mib":592.390625},{"t_s":56,"rss_mib":592.390625},{"t_s":57,"rss_mib":592.390625},{"t_s":57,"rss_mib":592.390625},{"t_s":58,"rss_mib":592.390625},{"t_s":59,"rss_mib":592.390625},{"t_s":59,"rss_mib":592.390625},{"t_s":59,"rss_mib":592.390625}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.018903158,"cost":6.092956223,"memory":440.350267221,"streaming":0.000002356,"streams_sustained":0.000001679,"throughput":647.307681345},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here"}}},"gemini":{"served":true,"reason":null,"status":"","path":"/v1beta/models/gpt-4o-mini-gemini:generateContent","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":10073,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":10889,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":10916,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":10236.486486486487,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":97.6897689768977,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2501839420461313,"certified":true,"suppressed":false},"cost_window_ok":{"value":592,"certified":true,"suppressed":false},"cost_window_rps":{"value":98,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.048986486486486486,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":101,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21419,"first_disqualified_conc":8,"lower_bound":false,"samples":612},{"bound_ms":100,"rps":{"value":101,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21419,"first_disqualified_conc":16,"lower_bound":false,"samples":612},{"bound_ms":null,"rps":{"value":101,"certified":true,"suppressed":false},"concurrency":2,"p99_us":21419,"first_disqualified_conc":4096,"lower_bound":false,"samples":612}],"sweep":[{"conc":1,"ok":602,"rps":100,"p99_us":10690,"fail":0},{"conc":1,"ok":602,"rps":100,"p99_us":10733,"fail":0},{"conc":1,"ok":602,"rps":100,"p99_us":10715,"fail":0},{"conc":2,"ok":612,"rps":101,"p99_us":21426,"fail":0},{"conc":2,"ok":612,"rps":101,"p99_us":21419,"fail":0},{"conc":2,"ok":606,"rps":100,"p99_us":20840,"fail":0},{"conc":4,"ok":606,"rps":100,"p99_us":43127,"fail":0},{"conc":4,"ok":605,"rps":100,"p99_us":43139,"fail":0},{"conc":4,"ok":607,"rps":101,"p99_us":42577,"fail":0},{"conc":8,"ok":605,"rps":100,"p99_us":85937,"fail":0},{"conc":8,"ok":603,"rps":100,"p99_us":86065,"fail":0},{"conc":8,"ok":606,"rps":100,"p99_us":85815,"fail":0},{"conc":16,"ok":604,"rps":100,"p99_us":178499,"fail":0},{"conc":16,"ok":601,"rps":100,"p99_us":174225,"fail":0},{"conc":16,"ok":603,"rps":100,"p99_us":173548,"fail":0},{"conc":32,"ok":604,"rps":100,"p99_us":416868,"fail":0},{"conc":32,"ok":601,"rps":100,"p99_us":499970,"fail":0},{"conc":32,"ok":603,"rps":100,"p99_us":520593,"fail":0},{"conc":64,"ok":588,"rps":97,"p99_us":1833009,"fail":0},{"conc":64,"ok":601,"rps":100,"p99_us":921334,"fail":0},{"conc":64,"ok":588,"rps":97,"p99_us":1767822,"fail":0},{"conc":128,"ok":529,"rps":88,"p99_us":4891289,"fail":0},{"conc":128,"ok":585,"rps":97,"p99_us":1391312,"fail":0},{"conc":128,"ok":509,"rps":84,"p99_us":5474979,"fail":0},{"conc":256,"ok":558,"rps":92,"p99_us":5344441,"fail":0},{"conc":256,"ok":556,"rps":92,"p99_us":5347906,"fail":0},{"conc":256,"ok":512,"rps":85,"p99_us":2713161,"fail":0},{"conc":512,"ok":555,"rps":92,"p99_us":5504849,"fail":0},{"conc":512,"ok":390,"rps":64,"p99_us":5169981,"fail":0},{"conc":512,"ok":514,"rps":85,"p99_us":4309556,"fail":0},{"conc":1024,"ok":344,"rps":57,"p99_us":4871898,"fail":0},{"conc":1024,"ok":359,"rps":59,"p99_us":4240715,"fail":0},{"conc":1024,"ok":356,"rps":59,"p99_us":4850758,"fail":0},{"conc":2048,"ok":384,"rps":63,"p99_us":4679605,"fail":0},{"conc":2048,"ok":375,"rps":61,"p99_us":4418040,"fail":0},{"conc":2048,"ok":384,"rps":63,"p99_us":5396013,"fail":0},{"conc":4096,"ok":367,"rps":61,"p99_us":4693481,"fail":6},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":1159}],"rungs":[{"conc":1,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":10715,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":101,"certified":true,"suppressed":false},"p99_us":21419,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":43127,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":85937,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":174225,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":100,"certified":true,"suppressed":false},"p99_us":499970,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":1767822,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":88,"certified":true,"suppressed":false},"p99_us":4891289,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":92,"certified":true,"suppressed":false},"p99_us":5344441,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":85,"certified":true,"suppressed":false},"p99_us":5169981,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":59,"certified":true,"suppressed":false},"p99_us":4850758,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":63,"certified":true,"suppressed":false},"p99_us":4679605,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=4096 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"gemini>gemini is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 592 successful gateway round trip(s) and 267671 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":592,"direct_c1_samples":267671},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.546875,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":627.6875,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":627.6875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":627.6875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":627.6875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.546875},{"t_s":2,"rss_mib":625.11328125},{"t_s":2,"rss_mib":625.125},{"t_s":3,"rss_mib":625.1484375},{"t_s":6,"rss_mib":625.1484375},{"t_s":6,"rss_mib":625.15234375},{"t_s":9,"rss_mib":625.15234375},{"t_s":10,"rss_mib":625.1953125},{"t_s":12,"rss_mib":625.1953125},{"t_s":13,"rss_mib":625.21484375},{"t_s":15,"rss_mib":625.21484375},{"t_s":17,"rss_mib":625.33984375},{"t_s":18,"rss_mib":625.33984375},{"t_s":20,"rss_mib":625.375},{"t_s":21,"rss_mib":625.375},{"t_s":23,"rss_mib":625.58984375},{"t_s":24,"rss_mib":625.58984375},{"t_s":26,"rss_mib":625.609375},{"t_s":27,"rss_mib":625.609375},{"t_s":29,"rss_mib":625.78125},{"t_s":30,"rss_mib":625.78125},{"t_s":32,"rss_mib":625.78515625},{"t_s":33,"rss_mib":625.78515625},{"t_s":35,"rss_mib":625.87890625},{"t_s":36,"rss_mib":625.87890625},{"t_s":38,"rss_mib":625.8828125},{"t_s":39,"rss_mib":625.8828125},{"t_s":42,"rss_mib":625.96875},{"t_s":42,"rss_mib":625.96875},{"t_s":44,"rss_mib":625.97265625},{"t_s":45,"rss_mib":625.97265625},{"t_s":48,"rss_mib":626.08984375},{"t_s":48,"rss_mib":626.09375},{"t_s":50,"rss_mib":626.171875},{"t_s":51,"rss_mib":626.171875},{"t_s":54,"rss_mib":626.17578125},{"t_s":54,"rss_mib":626.19140625},{"t_s":57,"rss_mib":626.3828125},{"t_s":57,"rss_mib":626.3828125},{"t_s":60,"rss_mib":626.3828125},{"t_s":62,"rss_mib":626.39453125},{"t_s":63,"rss_mib":626.39453125},{"t_s":66,"rss_mib":626.40625},{"t_s":66,"rss_mib":626.40625},{"t_s":69,"rss_mib":626.4140625},{"t_s":69,"rss_mib":626.4140625},{"t_s":72,"rss_mib":626.453125},{"t_s":72,"rss_mib":626.453125},{"t_s":74,"rss_mib":626.46484375},{"t_s":75,"rss_mib":626.46484375},{"t_s":77,"rss_mib":626.57421875},{"t_s":78,"rss_mib":626.57421875},{"t_s":80,"rss_mib":626.625},{"t_s":81,"rss_mib":626.625},{"t_s":84,"rss_mib":626.7421875},{"t_s":84,"rss_mib":626.7421875},{"t_s":85,"rss_mib":626.75},{"t_s":87,"rss_mib":626.75},{"t_s":89,"rss_mib":626.796875},{"t_s":90,"rss_mib":626.796875},{"t_s":92,"rss_mib":626.80859375},{"t_s":93,"rss_mib":626.80859375},{"t_s":96,"rss_mib":626.80859375},{"t_s":99,"rss_mib":626.80859375},{"t_s":102,"rss_mib":626.80859375},{"t_s":105,"rss_mib":626.80859375},{"t_s":108,"rss_mib":626.80859375},{"t_s":111,"rss_mib":626.80859375},{"t_s":114,"rss_mib":626.80859375},{"t_s":117,"rss_mib":626.80859375},{"t_s":121,"rss_mib":626.80859375},{"t_s":123,"rss_mib":626.80859375},{"t_s":127,"rss_mib":626.80859375},{"t_s":130,"rss_mib":626.80859375},{"t_s":133,"rss_mib":626.80859375},{"t_s":136,"rss_mib":626.80859375},{"t_s":139,"rss_mib":626.80859375},{"t_s":142,"rss_mib":626.80859375},{"t_s":145,"rss_mib":626.80859375},{"t_s":148,"rss_mib":626.80859375},{"t_s":151,"rss_mib":626.80859375},{"t_s":154,"rss_mib":626.80859375},{"t_s":157,"rss_mib":626.80859375},{"t_s":160,"rss_mib":626.80859375},{"t_s":162,"rss_mib":626.81640625},{"t_s":163,"rss_mib":626.81640625},{"t_s":166,"rss_mib":626.828125},{"t_s":166,"rss_mib":626.83203125},{"t_s":166,"rss_mib":626.84375},{"t_s":169,"rss_mib":626.84375},{"t_s":172,"rss_mib":626.8515625},{"t_s":172,"rss_mib":626.85546875},{"t_s":172,"rss_mib":626.87890625},{"t_s":175,"rss_mib":626.87890625},{"t_s":178,"rss_mib":626.88671875},{"t_s":178,"rss_mib":626.88671875},{"t_s":180,"rss_mib":626.92578125},{"t_s":181,"rss_mib":626.92578125},{"t_s":181,"rss_mib":626.9296875},{"t_s":184,"rss_mib":626.9296875},{"t_s":185,"rss_mib":626.96484375},{"t_s":187,"rss_mib":626.96484375},{"t_s":190,"rss_mib":627.0078125},{"t_s":190,"rss_mib":627.0078125},{"t_s":193,"rss_mib":627.140625},{"t_s":193,"rss_mib":627.140625},{"t_s":195,"rss_mib":627.16015625},{"t_s":196,"rss_mib":627.16015625},{"t_s":197,"rss_mib":627.25},{"t_s":199,"rss_mib":627.25},{"t_s":201,"rss_mib":627.26171875},{"t_s":202,"rss_mib":627.26171875},{"t_s":205,"rss_mib":627.26171875},{"t_s":208,"rss_mib":627.33984375},{"t_s":208,"rss_mib":627.33984375},{"t_s":211,"rss_mib":627.33984375},{"t_s":214,"rss_mib":627.52734375},{"t_s":214,"rss_mib":627.52734375},{"t_s":216,"rss_mib":627.53125},{"t_s":217,"rss_mib":627.53125},{"t_s":220,"rss_mib":627.66015625},{"t_s":220,"rss_mib":627.66015625},{"t_s":223,"rss_mib":627.6875},{"t_s":223,"rss_mib":627.6875},{"t_s":226,"rss_mib":627.6875},{"t_s":229,"rss_mib":627.6875},{"t_s":232,"rss_mib":627.6875},{"t_s":235,"rss_mib":627.6875},{"t_s":239,"rss_mib":627.6875},{"t_s":242,"rss_mib":627.6875},{"t_s":245,"rss_mib":627.6875},{"t_s":248,"rss_mib":627.6875},{"t_s":251,"rss_mib":627.6875},{"t_s":254,"rss_mib":627.6875},{"t_s":257,"rss_mib":627.6875},{"t_s":260,"rss_mib":627.6875},{"t_s":263,"rss_mib":627.6875},{"t_s":266,"rss_mib":627.6875},{"t_s":269,"rss_mib":627.6875},{"t_s":272,"rss_mib":627.6875},{"t_s":275,"rss_mib":627.6875},{"t_s":278,"rss_mib":627.6875},{"t_s":281,"rss_mib":627.6875},{"t_s":284,"rss_mib":627.6875},{"t_s":287,"rss_mib":627.6875},{"t_s":290,"rss_mib":627.6875},{"t_s":293,"rss_mib":627.6875},{"t_s":296,"rss_mib":627.6875},{"t_s":299,"rss_mib":627.6875},{"t_s":302,"rss_mib":627.6875},{"t_s":305,"rss_mib":627.6875},{"t_s":308,"rss_mib":627.6875},{"t_s":311,"rss_mib":627.6875},{"t_s":314,"rss_mib":627.6875},{"t_s":317,"rss_mib":627.6875},{"t_s":320,"rss_mib":627.6875},{"t_s":323,"rss_mib":627.6875},{"t_s":326,"rss_mib":627.6875},{"t_s":329,"rss_mib":627.6875},{"t_s":332,"rss_mib":627.6875},{"t_s":335,"rss_mib":627.6875},{"t_s":338,"rss_mib":627.6875},{"t_s":341,"rss_mib":627.6875},{"t_s":344,"rss_mib":627.6875},{"t_s":347,"rss_mib":627.6875},{"t_s":350,"rss_mib":627.6875},{"t_s":353,"rss_mib":627.6875},{"t_s":356,"rss_mib":627.6875},{"t_s":359,"rss_mib":627.6875},{"t_s":362,"rss_mib":627.6875}],"idle_rss_series":[{"t_s":0,"rss_mib":591.546875},{"t_s":0,"rss_mib":591.546875},{"t_s":0,"rss_mib":591.546875},{"t_s":1,"rss_mib":591.546875},{"t_s":1,"rss_mib":591.546875},{"t_s":2,"rss_mib":591.546875},{"t_s":2,"rss_mib":591.546875},{"t_s":3,"rss_mib":591.546875},{"t_s":3,"rss_mib":591.546875},{"t_s":4,"rss_mib":591.546875},{"t_s":5,"rss_mib":591.546875},{"t_s":5,"rss_mib":591.546875},{"t_s":5,"rss_mib":591.546875},{"t_s":6,"rss_mib":591.546875},{"t_s":6,"rss_mib":591.546875},{"t_s":7,"rss_mib":591.546875},{"t_s":7,"rss_mib":591.546875},{"t_s":8,"rss_mib":591.546875},{"t_s":8,"rss_mib":591.546875},{"t_s":9,"rss_mib":591.546875},{"t_s":10,"rss_mib":591.546875},{"t_s":10,"rss_mib":591.546875},{"t_s":10,"rss_mib":591.546875},{"t_s":11,"rss_mib":591.546875},{"t_s":11,"rss_mib":591.546875},{"t_s":12,"rss_mib":591.546875},{"t_s":12,"rss_mib":591.546875},{"t_s":13,"rss_mib":591.546875},{"t_s":13,"rss_mib":591.546875},{"t_s":14,"rss_mib":591.546875},{"t_s":15,"rss_mib":591.546875},{"t_s":15,"rss_mib":591.546875},{"t_s":15,"rss_mib":591.546875},{"t_s":16,"rss_mib":591.546875},{"t_s":16,"rss_mib":591.546875},{"t_s":17,"rss_mib":591.546875},{"t_s":17,"rss_mib":591.546875},{"t_s":18,"rss_mib":591.546875},{"t_s":18,"rss_mib":591.546875},{"t_s":19,"rss_mib":591.546875},{"t_s":20,"rss_mib":591.546875},{"t_s":20,"rss_mib":591.546875},{"t_s":20,"rss_mib":591.546875},{"t_s":21,"rss_mib":591.546875},{"t_s":21,"rss_mib":591.546875},{"t_s":22,"rss_mib":591.546875},{"t_s":22,"rss_mib":591.546875},{"t_s":23,"rss_mib":591.546875},{"t_s":23,"rss_mib":591.546875},{"t_s":24,"rss_mib":591.546875},{"t_s":25,"rss_mib":591.546875},{"t_s":25,"rss_mib":591.546875},{"t_s":25,"rss_mib":591.546875},{"t_s":26,"rss_mib":591.546875},{"t_s":26,"rss_mib":591.546875},{"t_s":27,"rss_mib":591.546875},{"t_s":27,"rss_mib":591.546875},{"t_s":28,"rss_mib":591.546875},{"t_s":28,"rss_mib":591.546875},{"t_s":29,"rss_mib":591.546875},{"t_s":30,"rss_mib":591.546875},{"t_s":30,"rss_mib":591.546875},{"t_s":30,"rss_mib":591.546875},{"t_s":31,"rss_mib":591.546875},{"t_s":31,"rss_mib":591.546875},{"t_s":32,"rss_mib":591.546875},{"t_s":32,"rss_mib":591.546875},{"t_s":33,"rss_mib":591.546875},{"t_s":33,"rss_mib":591.546875},{"t_s":34,"rss_mib":591.546875},{"t_s":35,"rss_mib":591.546875},{"t_s":35,"rss_mib":591.546875},{"t_s":35,"rss_mib":591.546875},{"t_s":36,"rss_mib":591.546875},{"t_s":36,"rss_mib":591.546875},{"t_s":37,"rss_mib":591.546875},{"t_s":37,"rss_mib":591.546875},{"t_s":38,"rss_mib":591.546875},{"t_s":38,"rss_mib":591.546875},{"t_s":39,"rss_mib":591.546875},{"t_s":40,"rss_mib":591.546875},{"t_s":40,"rss_mib":591.546875},{"t_s":40,"rss_mib":591.546875},{"t_s":41,"rss_mib":591.546875},{"t_s":41,"rss_mib":591.546875},{"t_s":42,"rss_mib":591.546875},{"t_s":42,"rss_mib":591.546875},{"t_s":43,"rss_mib":591.546875},{"t_s":44,"rss_mib":591.546875},{"t_s":44,"rss_mib":591.546875},{"t_s":45,"rss_mib":591.546875},{"t_s":45,"rss_mib":591.546875},{"t_s":45,"rss_mib":591.546875},{"t_s":46,"rss_mib":591.546875},{"t_s":46,"rss_mib":591.546875},{"t_s":47,"rss_mib":591.546875},{"t_s":47,"rss_mib":591.546875},{"t_s":48,"rss_mib":591.546875},{"t_s":49,"rss_mib":591.546875},{"t_s":49,"rss_mib":591.546875},{"t_s":50,"rss_mib":591.546875},{"t_s":50,"rss_mib":591.546875},{"t_s":50,"rss_mib":591.546875},{"t_s":51,"rss_mib":591.546875},{"t_s":51,"rss_mib":591.546875},{"t_s":52,"rss_mib":591.546875},{"t_s":52,"rss_mib":591.546875},{"t_s":53,"rss_mib":591.546875},{"t_s":54,"rss_mib":591.546875},{"t_s":54,"rss_mib":591.546875},{"t_s":55,"rss_mib":591.546875},{"t_s":55,"rss_mib":591.546875},{"t_s":55,"rss_mib":591.546875},{"t_s":56,"rss_mib":591.546875},{"t_s":56,"rss_mib":591.546875},{"t_s":57,"rss_mib":591.546875},{"t_s":57,"rss_mib":591.546875},{"t_s":58,"rss_mib":591.546875},{"t_s":59,"rss_mib":591.546875},{"t_s":59,"rss_mib":591.546875},{"t_s":59,"rss_mib":591.546875}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.010550945,"cost":6.068859048,"memory":438.451203867,"streaming":0.000001868,"streams_sustained":0.000001752,"throughput":419.692569826},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.10ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.10ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c4096.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}},"cohere":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/v2/chat","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"bedrock":{"served":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","status":"","path":"/model/gpt-4o-mini-gemini/converse","verdict_note":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":null,"stream":null,"memory":null,"timings_s":null,"upstream_transport":null,"absences":{}},"openai":{"served":true,"reason":null,"status":"","path":"/v1/chat/completions","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":8225,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":8995,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":9023,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":8281.036834924966,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":120.75782537067545,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25021912323444456,"certified":true,"suppressed":false},"cost_window_ok":{"value":733,"certified":true,"suppressed":false},"cost_window_rps":{"value":122,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.0286493860845839,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":121,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8923,"first_disqualified_conc":2,"lower_bound":false,"samples":732},{"bound_ms":50,"rps":{"value":123,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17053,"first_disqualified_conc":8,"lower_bound":false,"samples":740},{"bound_ms":100,"rps":{"value":123,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17053,"first_disqualified_conc":16,"lower_bound":false,"samples":740},{"bound_ms":null,"rps":{"value":127,"certified":true,"suppressed":false},"concurrency":128,"p99_us":1100645,"first_disqualified_conc":8192,"lower_bound":false,"samples":767}],"sweep":[{"conc":1,"ok":732,"rps":121,"p99_us":8927,"fail":0},{"conc":1,"ok":732,"rps":121,"p99_us":8946,"fail":0},{"conc":1,"ok":732,"rps":121,"p99_us":8923,"fail":0},{"conc":2,"ok":738,"rps":122,"p99_us":17146,"fail":0},{"conc":2,"ok":738,"rps":122,"p99_us":17029,"fail":0},{"conc":2,"ok":740,"rps":123,"p99_us":17053,"fail":0},{"conc":4,"ok":737,"rps":121,"p99_us":34699,"fail":0},{"conc":4,"ok":741,"rps":123,"p99_us":34173,"fail":0},{"conc":4,"ok":735,"rps":122,"p99_us":34493,"fail":0},{"conc":8,"ok":737,"rps":121,"p99_us":70600,"fail":0},{"conc":8,"ok":736,"rps":122,"p99_us":69956,"fail":0},{"conc":8,"ok":735,"rps":122,"p99_us":71037,"fail":0},{"conc":16,"ok":741,"rps":123,"p99_us":143457,"fail":0},{"conc":16,"ok":739,"rps":123,"p99_us":142716,"fail":0},{"conc":16,"ok":739,"rps":123,"p99_us":141054,"fail":0},{"conc":32,"ok":745,"rps":124,"p99_us":286246,"fail":0},{"conc":32,"ok":743,"rps":123,"p99_us":369992,"fail":0},{"conc":32,"ok":741,"rps":123,"p99_us":369450,"fail":0},{"conc":64,"ok":759,"rps":126,"p99_us":545933,"fail":0},{"conc":64,"ok":684,"rps":113,"p99_us":1838433,"fail":0},{"conc":64,"ok":736,"rps":122,"p99_us":1351205,"fail":0},{"conc":128,"ok":643,"rps":107,"p99_us":1490850,"fail":0},{"conc":128,"ok":767,"rps":127,"p99_us":1100645,"fail":0},{"conc":128,"ok":660,"rps":109,"p99_us":5512020,"fail":0},{"conc":256,"ok":618,"rps":102,"p99_us":2603559,"fail":0},{"conc":256,"ok":637,"rps":106,"p99_us":5083390,"fail":0},{"conc":256,"ok":616,"rps":102,"p99_us":2515573,"fail":0},{"conc":512,"ok":711,"rps":117,"p99_us":5375726,"fail":0},{"conc":512,"ok":655,"rps":109,"p99_us":5463306,"fail":0},{"conc":512,"ok":661,"rps":110,"p99_us":5000139,"fail":0},{"conc":1024,"ok":682,"rps":113,"p99_us":5168617,"fail":0},{"conc":1024,"ok":714,"rps":118,"p99_us":5459430,"fail":0},{"conc":1024,"ok":690,"rps":114,"p99_us":4116320,"fail":0},{"conc":2048,"ok":643,"rps":107,"p99_us":3734467,"fail":0},{"conc":2048,"ok":707,"rps":117,"p99_us":5358890,"fail":0},{"conc":2048,"ok":648,"rps":107,"p99_us":3819899,"fail":0},{"conc":4096,"ok":713,"rps":118,"p99_us":4102387,"fail":538},{"conc":4096,"ok":702,"rps":116,"p99_us":5288371,"fail":0},{"conc":4096,"ok":1,"rps":0.16663404563603973,"p99_us":5932560,"fail":0},{"conc":8192,"ok":526,"rps":87,"p99_us":5569789,"fail":125},{"conc":8192,"ok":20,"rps":3,"p99_us":5841381,"fail":4102},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0}],"rungs":[{"conc":1,"rps":{"value":121,"certified":true,"suppressed":false},"p99_us":8927,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":17053,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":34493,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":70600,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":123,"certified":true,"suppressed":false},"p99_us":142716,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":123,"certified":true,"suppressed":false},"p99_us":369450,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":1351205,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":109,"certified":true,"suppressed":false},"p99_us":1490850,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":2603559,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":110,"certified":true,"suppressed":false},"p99_us":5375726,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":114,"certified":true,"suppressed":false},"p99_us":5168617,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":107,"certified":true,"suppressed":false},"p99_us":3819899,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":0.16663404563603973,"certified":true,"suppressed":false},"p99_us":5288371,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no window at c=8192 was clean: 2 of 3 failed requests, the rest carried no usable rate or failure count"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 725 successful gateway round trip(s) and 265737 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":725,"direct_c1_samples":265737},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":591.8359375,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":665.40625,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":665.40625,"certified":true,"suppressed":false},"peak_rss_mib":{"value":665.40625,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":665.40625,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":591.8359375},{"t_s":1,"rss_mib":665.40625},{"t_s":2,"rss_mib":665.40625},{"t_s":6,"rss_mib":665.40625},{"t_s":8,"rss_mib":665.40625},{"t_s":12,"rss_mib":665.40625},{"t_s":15,"rss_mib":665.40625},{"t_s":18,"rss_mib":665.40625},{"t_s":21,"rss_mib":665.40625},{"t_s":24,"rss_mib":665.40625},{"t_s":27,"rss_mib":665.40625},{"t_s":30,"rss_mib":665.40625},{"t_s":33,"rss_mib":665.40625},{"t_s":36,"rss_mib":665.40625},{"t_s":39,"rss_mib":665.40625},{"t_s":42,"rss_mib":665.40625},{"t_s":45,"rss_mib":665.40625},{"t_s":48,"rss_mib":665.40625},{"t_s":51,"rss_mib":665.40625},{"t_s":54,"rss_mib":665.40625},{"t_s":57,"rss_mib":665.40625},{"t_s":60,"rss_mib":665.40625},{"t_s":63,"rss_mib":665.40625},{"t_s":66,"rss_mib":665.40625},{"t_s":69,"rss_mib":665.40625},{"t_s":72,"rss_mib":665.40625},{"t_s":75,"rss_mib":665.40625},{"t_s":78,"rss_mib":665.40625},{"t_s":81,"rss_mib":665.40625},{"t_s":84,"rss_mib":665.40625},{"t_s":87,"rss_mib":665.40625},{"t_s":90,"rss_mib":665.40625},{"t_s":93,"rss_mib":665.40625},{"t_s":96,"rss_mib":665.40625},{"t_s":99,"rss_mib":665.40625},{"t_s":102,"rss_mib":665.40625},{"t_s":105,"rss_mib":665.40625},{"t_s":108,"rss_mib":665.40625},{"t_s":111,"rss_mib":665.40625},{"t_s":114,"rss_mib":665.40625},{"t_s":117,"rss_mib":665.40625},{"t_s":120,"rss_mib":665.40625},{"t_s":123,"rss_mib":665.40625},{"t_s":126,"rss_mib":665.40625},{"t_s":129,"rss_mib":665.40625},{"t_s":132,"rss_mib":665.40625},{"t_s":135,"rss_mib":665.40625},{"t_s":138,"rss_mib":665.40625},{"t_s":141,"rss_mib":665.40625},{"t_s":144,"rss_mib":665.40625},{"t_s":147,"rss_mib":665.40625},{"t_s":150,"rss_mib":665.40625},{"t_s":154,"rss_mib":665.40625},{"t_s":156,"rss_mib":665.40625},{"t_s":160,"rss_mib":665.40625},{"t_s":162,"rss_mib":665.40625},{"t_s":166,"rss_mib":665.40625},{"t_s":169,"rss_mib":665.40625},{"t_s":172,"rss_mib":665.40625},{"t_s":175,"rss_mib":665.40625},{"t_s":178,"rss_mib":665.40625},{"t_s":181,"rss_mib":665.40625},{"t_s":184,"rss_mib":665.40625},{"t_s":187,"rss_mib":665.40625},{"t_s":190,"rss_mib":665.40625},{"t_s":193,"rss_mib":665.40625},{"t_s":196,"rss_mib":665.40625},{"t_s":199,"rss_mib":665.40625},{"t_s":202,"rss_mib":665.40625},{"t_s":205,"rss_mib":665.40625},{"t_s":208,"rss_mib":665.40625},{"t_s":211,"rss_mib":665.40625},{"t_s":214,"rss_mib":665.40625},{"t_s":217,"rss_mib":665.40625},{"t_s":220,"rss_mib":665.40625},{"t_s":223,"rss_mib":665.40625},{"t_s":226,"rss_mib":665.40625},{"t_s":229,"rss_mib":665.40625},{"t_s":232,"rss_mib":665.40625},{"t_s":235,"rss_mib":665.40625},{"t_s":238,"rss_mib":665.40625},{"t_s":241,"rss_mib":665.40625},{"t_s":244,"rss_mib":665.40625},{"t_s":247,"rss_mib":665.40625},{"t_s":250,"rss_mib":665.40625},{"t_s":253,"rss_mib":665.40625},{"t_s":256,"rss_mib":665.40625},{"t_s":259,"rss_mib":665.40625},{"t_s":262,"rss_mib":665.40625},{"t_s":265,"rss_mib":665.40625},{"t_s":268,"rss_mib":665.40625},{"t_s":271,"rss_mib":665.40625},{"t_s":274,"rss_mib":665.40625},{"t_s":277,"rss_mib":665.40625},{"t_s":280,"rss_mib":665.40625},{"t_s":283,"rss_mib":665.40625},{"t_s":286,"rss_mib":665.40625},{"t_s":289,"rss_mib":665.40625},{"t_s":292,"rss_mib":665.40625},{"t_s":295,"rss_mib":665.40625},{"t_s":298,"rss_mib":665.40625},{"t_s":301,"rss_mib":665.40625},{"t_s":304,"rss_mib":665.40625},{"t_s":307,"rss_mib":665.40625},{"t_s":310,"rss_mib":665.40625},{"t_s":314,"rss_mib":665.40625},{"t_s":316,"rss_mib":665.40625},{"t_s":320,"rss_mib":665.40625},{"t_s":322,"rss_mib":665.40625},{"t_s":326,"rss_mib":665.40625},{"t_s":329,"rss_mib":665.40625},{"t_s":332,"rss_mib":665.40625},{"t_s":335,"rss_mib":665.40625},{"t_s":338,"rss_mib":665.40625},{"t_s":341,"rss_mib":665.40625},{"t_s":344,"rss_mib":665.40625},{"t_s":347,"rss_mib":665.40625},{"t_s":350,"rss_mib":665.40625},{"t_s":353,"rss_mib":665.40625},{"t_s":356,"rss_mib":665.40625},{"t_s":359,"rss_mib":665.40625},{"t_s":362,"rss_mib":665.40625}],"idle_rss_series":[{"t_s":0,"rss_mib":591.8359375},{"t_s":0,"rss_mib":591.8359375},{"t_s":0,"rss_mib":591.8359375},{"t_s":1,"rss_mib":591.8359375},{"t_s":1,"rss_mib":591.8359375},{"t_s":2,"rss_mib":591.8359375},{"t_s":2,"rss_mib":591.8359375},{"t_s":3,"rss_mib":591.8359375},{"t_s":3,"rss_mib":591.8359375},{"t_s":4,"rss_mib":591.8359375},{"t_s":5,"rss_mib":591.8359375},{"t_s":5,"rss_mib":591.8359375},{"t_s":5,"rss_mib":591.8359375},{"t_s":6,"rss_mib":591.8359375},{"t_s":6,"rss_mib":591.8359375},{"t_s":7,"rss_mib":591.8359375},{"t_s":7,"rss_mib":591.8359375},{"t_s":8,"rss_mib":591.8359375},{"t_s":8,"rss_mib":591.8359375},{"t_s":9,"rss_mib":591.8359375},{"t_s":10,"rss_mib":591.8359375},{"t_s":10,"rss_mib":591.8359375},{"t_s":10,"rss_mib":591.8359375},{"t_s":11,"rss_mib":591.8359375},{"t_s":11,"rss_mib":591.8359375},{"t_s":12,"rss_mib":591.8359375},{"t_s":12,"rss_mib":591.8359375},{"t_s":13,"rss_mib":591.8359375},{"t_s":13,"rss_mib":591.8359375},{"t_s":14,"rss_mib":591.8359375},{"t_s":15,"rss_mib":591.8359375},{"t_s":15,"rss_mib":591.8359375},{"t_s":15,"rss_mib":591.8359375},{"t_s":16,"rss_mib":591.8359375},{"t_s":16,"rss_mib":591.8359375},{"t_s":17,"rss_mib":591.8359375},{"t_s":17,"rss_mib":591.8359375},{"t_s":18,"rss_mib":591.8359375},{"t_s":18,"rss_mib":591.8359375},{"t_s":19,"rss_mib":591.8359375},{"t_s":20,"rss_mib":591.8359375},{"t_s":20,"rss_mib":591.8359375},{"t_s":20,"rss_mib":591.8359375},{"t_s":21,"rss_mib":591.8359375},{"t_s":21,"rss_mib":591.8359375},{"t_s":22,"rss_mib":591.8359375},{"t_s":22,"rss_mib":591.8359375},{"t_s":23,"rss_mib":591.8359375},{"t_s":23,"rss_mib":591.8359375},{"t_s":24,"rss_mib":591.8359375},{"t_s":25,"rss_mib":591.8359375},{"t_s":25,"rss_mib":591.8359375},{"t_s":25,"rss_mib":591.8359375},{"t_s":26,"rss_mib":591.8359375},{"t_s":26,"rss_mib":591.8359375},{"t_s":27,"rss_mib":591.8359375},{"t_s":27,"rss_mib":591.8359375},{"t_s":28,"rss_mib":591.8359375},{"t_s":28,"rss_mib":591.8359375},{"t_s":29,"rss_mib":591.8359375},{"t_s":30,"rss_mib":591.8359375},{"t_s":30,"rss_mib":591.8359375},{"t_s":30,"rss_mib":591.8359375},{"t_s":31,"rss_mib":591.8359375},{"t_s":31,"rss_mib":591.8359375},{"t_s":32,"rss_mib":591.8359375},{"t_s":32,"rss_mib":591.8359375},{"t_s":33,"rss_mib":591.8359375},{"t_s":33,"rss_mib":591.8359375},{"t_s":34,"rss_mib":591.8359375},{"t_s":35,"rss_mib":591.8359375},{"t_s":35,"rss_mib":591.8359375},{"t_s":35,"rss_mib":591.8359375},{"t_s":36,"rss_mib":591.8359375},{"t_s":36,"rss_mib":591.8359375},{"t_s":37,"rss_mib":591.8359375},{"t_s":37,"rss_mib":591.8359375},{"t_s":38,"rss_mib":591.8359375},{"t_s":38,"rss_mib":591.8359375},{"t_s":39,"rss_mib":591.8359375},{"t_s":40,"rss_mib":591.8359375},{"t_s":40,"rss_mib":591.8359375},{"t_s":40,"rss_mib":591.8359375},{"t_s":41,"rss_mib":591.8359375},{"t_s":41,"rss_mib":591.8359375},{"t_s":42,"rss_mib":591.8359375},{"t_s":42,"rss_mib":591.8359375},{"t_s":43,"rss_mib":591.8359375},{"t_s":43,"rss_mib":591.8359375},{"t_s":44,"rss_mib":591.8359375},{"t_s":45,"rss_mib":591.8359375},{"t_s":45,"rss_mib":591.8359375},{"t_s":45,"rss_mib":591.8359375},{"t_s":46,"rss_mib":591.8359375},{"t_s":46,"rss_mib":591.8359375},{"t_s":47,"rss_mib":591.8359375},{"t_s":47,"rss_mib":591.8359375},{"t_s":48,"rss_mib":591.8359375},{"t_s":48,"rss_mib":591.8359375},{"t_s":49,"rss_mib":591.8359375},{"t_s":50,"rss_mib":591.8359375},{"t_s":50,"rss_mib":591.8359375},{"t_s":50,"rss_mib":591.8359375},{"t_s":51,"rss_mib":591.8359375},{"t_s":51,"rss_mib":591.8359375},{"t_s":52,"rss_mib":591.8359375},{"t_s":52,"rss_mib":591.8359375},{"t_s":53,"rss_mib":591.8359375},{"t_s":53,"rss_mib":591.8359375},{"t_s":54,"rss_mib":591.8359375},{"t_s":55,"rss_mib":591.8359375},{"t_s":55,"rss_mib":591.8359375},{"t_s":55,"rss_mib":591.8359375},{"t_s":56,"rss_mib":591.8359375},{"t_s":56,"rss_mib":591.8359375},{"t_s":57,"rss_mib":591.8359375},{"t_s":57,"rss_mib":591.8359375},{"t_s":58,"rss_mib":591.8359375},{"t_s":58,"rss_mib":591.8359375},{"t_s":59,"rss_mib":591.8359375},{"t_s":59,"rss_mib":591.8359375}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.010592005,"cost":6.078295319,"memory":437.72641902,"streaming":0.000001964,"streams_sustained":0.000002177,"throughput":513.957992365},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}},"anthropic":{"served":true,"reason":null,"status":"","path":"/v1/messages","verdict_note":"","body_snippet":"","probe_note":null,"perf_dropped":null,"perf":{"added_latency_p50_us":{"value":8964,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":9686,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":9714,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":9144.144144144144,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":109.35960591133005,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.25073457424131973,"certified":true,"suppressed":false},"cost_window_ok":{"value":666,"certified":true,"suppressed":false},"cost_window_rps":{"value":110,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.03453453453453453,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":111,"certified":true,"suppressed":false},"concurrency":1,"p99_us":9727,"first_disqualified_conc":2,"lower_bound":false,"samples":667},{"bound_ms":50,"rps":{"value":113,"certified":true,"suppressed":false},"concurrency":2,"p99_us":19230,"first_disqualified_conc":8,"lower_bound":false,"samples":682},{"bound_ms":100,"rps":{"value":113,"certified":true,"suppressed":false},"concurrency":2,"p99_us":19230,"first_disqualified_conc":16,"lower_bound":false,"samples":682},{"bound_ms":null,"rps":{"value":113,"certified":true,"suppressed":false},"concurrency":2,"p99_us":19230,"first_disqualified_conc":16384,"lower_bound":false,"samples":682}],"sweep":[{"conc":1,"ok":666,"rps":110,"p99_us":9748,"fail":0},{"conc":1,"ok":666,"rps":110,"p99_us":9749,"fail":0},{"conc":1,"ok":667,"rps":111,"p99_us":9727,"fail":0},{"conc":2,"ok":672,"rps":111,"p99_us":19340,"fail":0},{"conc":2,"ok":682,"rps":113,"p99_us":19230,"fail":0},{"conc":2,"ok":672,"rps":111,"p99_us":19291,"fail":0},{"conc":4,"ok":665,"rps":109,"p99_us":38209,"fail":0},{"conc":4,"ok":668,"rps":111,"p99_us":37967,"fail":0},{"conc":4,"ok":667,"rps":111,"p99_us":38699,"fail":0},{"conc":8,"ok":667,"rps":111,"p99_us":76750,"fail":0},{"conc":8,"ok":669,"rps":111,"p99_us":77119,"fail":0},{"conc":8,"ok":670,"rps":111,"p99_us":76840,"fail":0},{"conc":16,"ok":671,"rps":111,"p99_us":154850,"fail":0},{"conc":16,"ok":671,"rps":111,"p99_us":153295,"fail":0},{"conc":16,"ok":674,"rps":112,"p99_us":153625,"fail":0},{"conc":32,"ok":620,"rps":103,"p99_us":853476,"fail":0},{"conc":32,"ok":674,"rps":112,"p99_us":423303,"fail":0},{"conc":32,"ok":673,"rps":112,"p99_us":423824,"fail":0},{"conc":64,"ok":667,"rps":111,"p99_us":1401405,"fail":0},{"conc":64,"ok":673,"rps":112,"p99_us":1405515,"fail":0},{"conc":64,"ok":663,"rps":110,"p99_us":1384933,"fail":0},{"conc":128,"ok":591,"rps":98,"p99_us":5559075,"fail":0},{"conc":128,"ok":649,"rps":108,"p99_us":5081473,"fail":0},{"conc":128,"ok":589,"rps":98,"p99_us":5584463,"fail":0},{"conc":256,"ok":640,"rps":105,"p99_us":5437980,"fail":0},{"conc":256,"ok":524,"rps":87,"p99_us":5363592,"fail":0},{"conc":256,"ok":579,"rps":96,"p99_us":5475628,"fail":0},{"conc":512,"ok":586,"rps":97,"p99_us":5330359,"fail":0},{"conc":512,"ok":587,"rps":97,"p99_us":5424123,"fail":0},{"conc":512,"ok":610,"rps":101,"p99_us":4691861,"fail":0},{"conc":1024,"ok":508,"rps":84,"p99_us":4651696,"fail":0},{"conc":1024,"ok":509,"rps":84,"p99_us":4753610,"fail":0},{"conc":1024,"ok":633,"rps":105,"p99_us":5411897,"fail":0},{"conc":2048,"ok":571,"rps":95,"p99_us":5503817,"fail":0},{"conc":2048,"ok":391,"rps":65,"p99_us":5221593,"fail":0},{"conc":2048,"ok":576,"rps":95,"p99_us":5424899,"fail":0},{"conc":4096,"ok":400,"rps":66,"p99_us":4917628,"fail":0},{"conc":4096,"ok":296,"rps":49,"p99_us":5811985,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":8192,"ok":318,"rps":52,"p99_us":5748769,"fail":156},{"conc":8192,"ok":377,"rps":61,"p99_us":5664295,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":4097},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":463},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":4329},{"conc":16384,"ok":249,"rps":41,"p99_us":5752766,"fail":2400}],"rungs":[{"conc":1,"rps":{"value":110,"certified":true,"suppressed":false},"p99_us":9748,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":111,"certified":true,"suppressed":false},"p99_us":19291,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":111,"certified":true,"suppressed":false},"p99_us":38209,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":111,"certified":true,"suppressed":false},"p99_us":76840,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":111,"certified":true,"suppressed":false},"p99_us":153625,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":112,"certified":true,"suppressed":false},"p99_us":423824,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":111,"certified":true,"suppressed":false},"p99_us":1401405,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":98,"certified":true,"suppressed":false},"p99_us":5559075,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":96,"certified":true,"suppressed":false},"p99_us":5437980,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":97,"certified":true,"suppressed":false},"p99_us":5330359,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":84,"certified":true,"suppressed":false},"p99_us":4753610,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":95,"certified":true,"suppressed":false},"p99_us":5424899,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":49,"certified":true,"suppressed":false},"p99_us":4917628,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":61,"certified":true,"suppressed":false},"p99_us":5664295,"windows":3,"clean_windows":1},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 666 successful gateway round trip(s) and 263751 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":666,"direct_c1_samples":263751},"stream":{"stream_served":"untestable","stream_error":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","reason":"untestable","stream_c1_note":null,"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}},"memory":{"protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","served":true,"serve_error":"","load_recipe":null,"idle_rss_mib":{"value":593.02734375,"certified":true,"suppressed":false},"steady_state_rss_mib":{"value":666.55078125,"certified":true,"suppressed":false},"recovered_rss_mib":{"value":666.5546875,"certified":true,"suppressed":false},"peak_rss_mib":{"value":666.5546875,"certified":true,"suppressed":false},"peak_rss_hwm_mib":{"value":666.5546875,"certified":true,"suppressed":false},"plateaued":true,"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false},"growth_rate_mib_per_min":{"value":0.0057259229747044565,"certified":true,"suppressed":false},"load_s":{"value":302,"certified":true,"suppressed":false},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"idle_static":{"value":1,"certified":true,"suppressed":false},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false},"rss_series":[{"t_s":0,"rss_mib":593.02734375},{"t_s":2,"rss_mib":666.04296875},{"t_s":2,"rss_mib":666.04296875},{"t_s":5,"rss_mib":666.06640625},{"t_s":6,"rss_mib":666.08203125},{"t_s":8,"rss_mib":666.16796875},{"t_s":9,"rss_mib":666.171875},{"t_s":11,"rss_mib":666.28125},{"t_s":12,"rss_mib":666.28125},{"t_s":14,"rss_mib":666.3671875},{"t_s":15,"rss_mib":666.3671875},{"t_s":17,"rss_mib":666.4296875},{"t_s":18,"rss_mib":666.4296875},{"t_s":18,"rss_mib":666.4375},{"t_s":21,"rss_mib":666.4375},{"t_s":23,"rss_mib":666.45703125},{"t_s":24,"rss_mib":666.45703125},{"t_s":27,"rss_mib":666.45703125},{"t_s":30,"rss_mib":666.45703125},{"t_s":30,"rss_mib":666.46484375},{"t_s":33,"rss_mib":666.46484375},{"t_s":36,"rss_mib":666.46484375},{"t_s":39,"rss_mib":666.46484375},{"t_s":42,"rss_mib":666.46484375},{"t_s":43,"rss_mib":666.46875},{"t_s":45,"rss_mib":666.46875},{"t_s":45,"rss_mib":666.47265625},{"t_s":48,"rss_mib":666.47265625},{"t_s":50,"rss_mib":666.48046875},{"t_s":51,"rss_mib":666.48046875},{"t_s":54,"rss_mib":666.48046875},{"t_s":55,"rss_mib":666.484375},{"t_s":57,"rss_mib":666.484375},{"t_s":60,"rss_mib":666.48828125},{"t_s":60,"rss_mib":666.48828125},{"t_s":61,"rss_mib":666.4921875},{"t_s":63,"rss_mib":666.4921875},{"t_s":63,"rss_mib":666.49609375},{"t_s":66,"rss_mib":666.49609375},{"t_s":69,"rss_mib":666.49609375},{"t_s":72,"rss_mib":666.49609375},{"t_s":75,"rss_mib":666.49609375},{"t_s":78,"rss_mib":666.49609375},{"t_s":81,"rss_mib":666.49609375},{"t_s":83,"rss_mib":666.5},{"t_s":84,"rss_mib":666.5},{"t_s":85,"rss_mib":666.50390625},{"t_s":87,"rss_mib":666.50390625},{"t_s":88,"rss_mib":666.5078125},{"t_s":90,"rss_mib":666.5078125},{"t_s":91,"rss_mib":666.515625},{"t_s":93,"rss_mib":666.515625},{"t_s":96,"rss_mib":666.515625},{"t_s":99,"rss_mib":666.515625},{"t_s":102,"rss_mib":666.515625},{"t_s":105,"rss_mib":666.515625},{"t_s":108,"rss_mib":666.515625},{"t_s":111,"rss_mib":666.515625},{"t_s":114,"rss_mib":666.515625},{"t_s":117,"rss_mib":666.515625},{"t_s":121,"rss_mib":666.515625},{"t_s":123,"rss_mib":666.51953125},{"t_s":123,"rss_mib":666.51953125},{"t_s":127,"rss_mib":666.51953125},{"t_s":130,"rss_mib":666.51953125},{"t_s":133,"rss_mib":666.51953125},{"t_s":136,"rss_mib":666.51953125},{"t_s":139,"rss_mib":666.51953125},{"t_s":142,"rss_mib":666.51953125},{"t_s":145,"rss_mib":666.51953125},{"t_s":148,"rss_mib":666.51953125},{"t_s":151,"rss_mib":666.51953125},{"t_s":154,"rss_mib":666.51953125},{"t_s":157,"rss_mib":666.51953125},{"t_s":160,"rss_mib":666.51953125},{"t_s":163,"rss_mib":666.51953125},{"t_s":166,"rss_mib":666.51953125},{"t_s":169,"rss_mib":666.51953125},{"t_s":172,"rss_mib":666.51953125},{"t_s":175,"rss_mib":666.51953125},{"t_s":178,"rss_mib":666.51953125},{"t_s":181,"rss_mib":666.51953125},{"t_s":184,"rss_mib":666.51953125},{"t_s":187,"rss_mib":666.51953125},{"t_s":190,"rss_mib":666.51953125},{"t_s":193,"rss_mib":666.51953125},{"t_s":196,"rss_mib":666.51953125},{"t_s":197,"rss_mib":666.5390625},{"t_s":199,"rss_mib":666.5390625},{"t_s":202,"rss_mib":666.5390625},{"t_s":205,"rss_mib":666.5390625},{"t_s":208,"rss_mib":666.5390625},{"t_s":210,"rss_mib":666.54296875},{"t_s":211,"rss_mib":666.54296875},{"t_s":214,"rss_mib":666.546875},{"t_s":214,"rss_mib":666.546875},{"t_s":217,"rss_mib":666.546875},{"t_s":220,"rss_mib":666.546875},{"t_s":222,"rss_mib":666.55078125},{"t_s":223,"rss_mib":666.55078125},{"t_s":226,"rss_mib":666.55078125},{"t_s":229,"rss_mib":666.55078125},{"t_s":232,"rss_mib":666.55078125},{"t_s":235,"rss_mib":666.55078125},{"t_s":238,"rss_mib":666.55078125},{"t_s":242,"rss_mib":666.55078125},{"t_s":245,"rss_mib":666.55078125},{"t_s":248,"rss_mib":666.55078125},{"t_s":251,"rss_mib":666.55078125},{"t_s":254,"rss_mib":666.55078125},{"t_s":257,"rss_mib":666.55078125},{"t_s":260,"rss_mib":666.55078125},{"t_s":263,"rss_mib":666.55078125},{"t_s":266,"rss_mib":666.55078125},{"t_s":269,"rss_mib":666.55078125},{"t_s":272,"rss_mib":666.55078125},{"t_s":275,"rss_mib":666.55078125},{"t_s":277,"rss_mib":666.5546875},{"t_s":278,"rss_mib":666.5546875},{"t_s":281,"rss_mib":666.5546875},{"t_s":284,"rss_mib":666.5546875},{"t_s":287,"rss_mib":666.5546875},{"t_s":290,"rss_mib":666.5546875},{"t_s":293,"rss_mib":666.5546875},{"t_s":296,"rss_mib":666.5546875},{"t_s":299,"rss_mib":666.5546875},{"t_s":302,"rss_mib":666.5546875},{"t_s":305,"rss_mib":666.5546875},{"t_s":308,"rss_mib":666.5546875},{"t_s":311,"rss_mib":666.5546875},{"t_s":314,"rss_mib":666.5546875},{"t_s":317,"rss_mib":666.5546875},{"t_s":320,"rss_mib":666.5546875},{"t_s":323,"rss_mib":666.5546875},{"t_s":326,"rss_mib":666.5546875},{"t_s":329,"rss_mib":666.5546875},{"t_s":332,"rss_mib":666.5546875},{"t_s":335,"rss_mib":666.5546875},{"t_s":338,"rss_mib":666.5546875},{"t_s":341,"rss_mib":666.5546875},{"t_s":344,"rss_mib":666.5546875},{"t_s":347,"rss_mib":666.5546875},{"t_s":350,"rss_mib":666.5546875},{"t_s":353,"rss_mib":666.5546875},{"t_s":356,"rss_mib":666.5546875},{"t_s":359,"rss_mib":666.5546875},{"t_s":362,"rss_mib":666.5546875}],"idle_rss_series":[{"t_s":0,"rss_mib":593.02734375},{"t_s":0,"rss_mib":593.02734375},{"t_s":0,"rss_mib":593.02734375},{"t_s":1,"rss_mib":593.02734375},{"t_s":1,"rss_mib":593.02734375},{"t_s":2,"rss_mib":593.02734375},{"t_s":2,"rss_mib":593.02734375},{"t_s":3,"rss_mib":593.02734375},{"t_s":3,"rss_mib":593.02734375},{"t_s":4,"rss_mib":593.02734375},{"t_s":5,"rss_mib":593.02734375},{"t_s":5,"rss_mib":593.02734375},{"t_s":5,"rss_mib":593.02734375},{"t_s":6,"rss_mib":593.02734375},{"t_s":6,"rss_mib":593.02734375},{"t_s":7,"rss_mib":593.02734375},{"t_s":7,"rss_mib":593.02734375},{"t_s":8,"rss_mib":593.02734375},{"t_s":8,"rss_mib":593.02734375},{"t_s":9,"rss_mib":593.02734375},{"t_s":10,"rss_mib":593.02734375},{"t_s":10,"rss_mib":593.02734375},{"t_s":10,"rss_mib":593.02734375},{"t_s":11,"rss_mib":593.02734375},{"t_s":11,"rss_mib":593.02734375},{"t_s":12,"rss_mib":593.02734375},{"t_s":12,"rss_mib":593.02734375},{"t_s":13,"rss_mib":593.02734375},{"t_s":13,"rss_mib":593.02734375},{"t_s":14,"rss_mib":593.02734375},{"t_s":15,"rss_mib":593.02734375},{"t_s":15,"rss_mib":593.02734375},{"t_s":15,"rss_mib":593.02734375},{"t_s":16,"rss_mib":593.02734375},{"t_s":16,"rss_mib":593.02734375},{"t_s":17,"rss_mib":593.02734375},{"t_s":17,"rss_mib":593.02734375},{"t_s":18,"rss_mib":593.02734375},{"t_s":18,"rss_mib":593.02734375},{"t_s":19,"rss_mib":593.02734375},{"t_s":20,"rss_mib":593.02734375},{"t_s":20,"rss_mib":593.02734375},{"t_s":20,"rss_mib":593.02734375},{"t_s":21,"rss_mib":593.02734375},{"t_s":21,"rss_mib":593.02734375},{"t_s":22,"rss_mib":593.02734375},{"t_s":22,"rss_mib":593.02734375},{"t_s":23,"rss_mib":593.02734375},{"t_s":23,"rss_mib":593.02734375},{"t_s":24,"rss_mib":593.02734375},{"t_s":25,"rss_mib":593.02734375},{"t_s":25,"rss_mib":593.02734375},{"t_s":25,"rss_mib":593.02734375},{"t_s":26,"rss_mib":593.02734375},{"t_s":26,"rss_mib":593.02734375},{"t_s":27,"rss_mib":593.02734375},{"t_s":27,"rss_mib":593.02734375},{"t_s":28,"rss_mib":593.02734375},{"t_s":28,"rss_mib":593.02734375},{"t_s":29,"rss_mib":593.02734375},{"t_s":30,"rss_mib":593.02734375},{"t_s":30,"rss_mib":593.02734375},{"t_s":30,"rss_mib":593.02734375},{"t_s":31,"rss_mib":593.02734375},{"t_s":31,"rss_mib":593.02734375},{"t_s":32,"rss_mib":593.02734375},{"t_s":32,"rss_mib":593.02734375},{"t_s":33,"rss_mib":593.02734375},{"t_s":33,"rss_mib":593.02734375},{"t_s":34,"rss_mib":593.02734375},{"t_s":35,"rss_mib":593.02734375},{"t_s":35,"rss_mib":593.02734375},{"t_s":35,"rss_mib":593.02734375},{"t_s":36,"rss_mib":593.02734375},{"t_s":36,"rss_mib":593.02734375},{"t_s":37,"rss_mib":593.02734375},{"t_s":37,"rss_mib":593.02734375},{"t_s":38,"rss_mib":593.02734375},{"t_s":38,"rss_mib":593.02734375},{"t_s":39,"rss_mib":593.02734375},{"t_s":40,"rss_mib":593.02734375},{"t_s":40,"rss_mib":593.02734375},{"t_s":40,"rss_mib":593.02734375},{"t_s":41,"rss_mib":593.02734375},{"t_s":41,"rss_mib":593.02734375},{"t_s":42,"rss_mib":593.02734375},{"t_s":42,"rss_mib":593.02734375},{"t_s":43,"rss_mib":593.02734375},{"t_s":43,"rss_mib":593.02734375},{"t_s":44,"rss_mib":593.02734375},{"t_s":45,"rss_mib":593.02734375},{"t_s":45,"rss_mib":593.02734375},{"t_s":45,"rss_mib":593.02734375},{"t_s":46,"rss_mib":593.02734375},{"t_s":46,"rss_mib":593.02734375},{"t_s":47,"rss_mib":593.02734375},{"t_s":47,"rss_mib":593.02734375},{"t_s":48,"rss_mib":593.02734375},{"t_s":48,"rss_mib":593.02734375},{"t_s":49,"rss_mib":593.02734375},{"t_s":50,"rss_mib":593.02734375},{"t_s":50,"rss_mib":593.02734375},{"t_s":50,"rss_mib":593.02734375},{"t_s":51,"rss_mib":593.02734375},{"t_s":51,"rss_mib":593.02734375},{"t_s":52,"rss_mib":593.02734375},{"t_s":52,"rss_mib":593.02734375},{"t_s":53,"rss_mib":593.02734375},{"t_s":53,"rss_mib":593.02734375},{"t_s":54,"rss_mib":593.02734375},{"t_s":55,"rss_mib":593.02734375},{"t_s":55,"rss_mib":593.02734375},{"t_s":55,"rss_mib":593.02734375},{"t_s":56,"rss_mib":593.02734375},{"t_s":56,"rss_mib":593.02734375},{"t_s":57,"rss_mib":593.02734375},{"t_s":57,"rss_mib":593.02734375},{"t_s":58,"rss_mib":593.02734375},{"t_s":59,"rss_mib":593.02734375},{"t_s":59,"rss_mib":593.02734375},{"t_s":59,"rss_mib":593.02734375}],"idle_window_s":60,"recovery_window_s":30},"timings_s":{"added_latency":12.013473803,"cost":6.085457139,"memory":438.43057722,"streaming":0.000002204,"streams_sustained":0.000001857,"throughput":630.508761122},"upstream_transport":"http","absences":{"memory.idle_shape":{"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe"},"memory.shape":{"reason":"not_measured","detail":"a settled window has no unsettled shape to describe"},"perf.frontier.1ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.1ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.concurrency":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.p99_us":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.rps":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.frontier.5ms.samples":{"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"perf.sweep.c16384.w1.p99_us":{"reason":"not_measured"},"perf.sweep.c16384.w2.p99_us":{"reason":"not_measured"},"perf.sweep.c4096.w3.p99_us":{"reason":"not_measured"},"perf.sweep.c8192.w3.p99_us":{"reason":"not_measured"},"stream.added_gap_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_gap_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p50_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.added_ttft_p99_us":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.gap_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_fps":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_headroom":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.streams_sustained_mock_ceiling":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_direct_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"},"stream.ttft_gw_samples":{"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here"}}}}}},"model":null,"upstream_endpoint":null,"ootb_config":null,"arch":"arm64","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","rig":{"arch":"arm64","release_url":"file:///home/ubuntu/benchmarking/rig","engine":{"commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","dirty":false},"mock":{"origin":"release","sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","asset_updated_at":null},"ugen":null,"box_qualify":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"somaxconn":65535,"tcp_max_syn_backlog":65535},"measured_at":"2026-11-10T17:39:41Z","started_at":null,"finished_at":null,"duration_s":null,"phase_s":null,"build_env":null},"matrix_from_snapshot":true,"snapshot_file":"result_litellm-python_2026-11-10T17-39-41Z.json","ootb_config":"model_list:\n  - model_name: gpt-4o-mini\n    litellm_params:\n      model: openai/gpt-4o-mini\n      use_chat_completions_api: true\n      api_base: http://127.0.0.1:8000/v1\n      api_key: dummy\n  - model_name: gpt-4o-mini-responses\n    litellm_params:\n      model: openai/responses/gpt-4o-mini\n      api_base: http://127.0.0.1:8000/v1\n      api_key: dummy\n  - model_name: gpt-4o-mini-anthropic\n    litellm_params:\n      model: anthropic/claude-3-5-sonnet-20241022\n      api_base: http://127.0.0.1:8000\n      api_key: dummy\n  - model_name: gpt-4o-mini-gemini\n    litellm_params:\n      model: gemini/gemini-1.5-flash\n      api_base: http://127.0.0.1:8000\n      api_key: dummy\n  - model_name: command-r\n    litellm_params:\n      model: cohere_chat/command-r\n      api_base: http://127.0.0.1:8000/v2/chat\n      api_key: dummy\n  - model_name: gpt-4o-mini-bedrock\n    litellm_params:\n      model: bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0\n      aws_bedrock_runtime_endpoint: http://127.0.0.1:8000\n      aws_access_key_id: AKIAMOCKACCESSKEY\n      aws_secret_access_key: mock-secret-access-key\n      aws_region_name: us-east-1\n\n# Public test token for the local mock rig, never a real provider credential.\ngeneral_settings:\n  master_key: sk-otb-local-mock-benchmark\n","best_cell":{"path":{"ingress":"openai","egress":"openai","dialect":"openai"},"source":{"kind":"matrix","sweep":"6x6-diagonal","build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","measured_at":"2026-11-10T17:39:41Z"},"added_latency_p50_us":{"value":8512,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":9522,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":9549,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":8644.793152639088,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":115.67656765676568,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.24990558455671374,"certified":true,"suppressed":false},"cost_window_ok":{"value":701,"certified":true,"suppressed":false},"cost_window_rps":{"value":116,"certified":true,"suppressed":false},"cost_threads":{"value":32,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.028530670470756064,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":117,"certified":true,"suppressed":false},"concurrency":1,"p99_us":9350,"first_disqualified_conc":2,"lower_bound":false,"samples":708},{"bound_ms":50,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":8,"lower_bound":false,"samples":718},{"bound_ms":100,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16,"lower_bound":false,"samples":718},{"bound_ms":null,"rps":{"value":119,"certified":true,"suppressed":false},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16384,"lower_bound":false,"samples":718}],"sweep":[{"conc":1,"ok":708,"rps":117,"p99_us":9178,"fail":0},{"conc":1,"ok":700,"rps":116,"p99_us":9310,"fail":0},{"conc":1,"ok":708,"rps":117,"p99_us":9350,"fail":0},{"conc":2,"ok":721,"rps":119,"p99_us":17781,"fail":0},{"conc":2,"ok":716,"rps":119,"p99_us":17810,"fail":0},{"conc":2,"ok":718,"rps":119,"p99_us":17670,"fail":0},{"conc":4,"ok":712,"rps":118,"p99_us":36003,"fail":0},{"conc":4,"ok":712,"rps":118,"p99_us":35805,"fail":0},{"conc":4,"ok":714,"rps":118,"p99_us":35485,"fail":0},{"conc":8,"ok":712,"rps":118,"p99_us":72900,"fail":0},{"conc":8,"ok":713,"rps":118,"p99_us":72270,"fail":0},{"conc":8,"ok":714,"rps":118,"p99_us":72240,"fail":0},{"conc":16,"ok":716,"rps":119,"p99_us":145952,"fail":0},{"conc":16,"ok":716,"rps":119,"p99_us":146710,"fail":0},{"conc":16,"ok":720,"rps":119,"p99_us":144717,"fail":0},{"conc":32,"ok":720,"rps":119,"p99_us":335625,"fail":0},{"conc":32,"ok":720,"rps":119,"p99_us":314783,"fail":0},{"conc":32,"ok":719,"rps":119,"p99_us":390651,"fail":0},{"conc":64,"ok":721,"rps":119,"p99_us":851275,"fail":0},{"conc":64,"ok":717,"rps":119,"p99_us":1384370,"fail":0},{"conc":64,"ok":719,"rps":119,"p99_us":568535,"fail":0},{"conc":128,"ok":689,"rps":114,"p99_us":5338973,"fail":0},{"conc":128,"ok":642,"rps":106,"p99_us":5507338,"fail":0},{"conc":128,"ok":640,"rps":105,"p99_us":1107255,"fail":0},{"conc":256,"ok":678,"rps":112,"p99_us":5466955,"fail":0},{"conc":256,"ok":630,"rps":104,"p99_us":4921753,"fail":0},{"conc":256,"ok":684,"rps":113,"p99_us":5494141,"fail":0},{"conc":512,"ok":646,"rps":107,"p99_us":4656402,"fail":0},{"conc":512,"ok":666,"rps":110,"p99_us":4568692,"fail":0},{"conc":512,"ok":674,"rps":112,"p99_us":5381288,"fail":0},{"conc":1024,"ok":520,"rps":86,"p99_us":4432245,"fail":0},{"conc":1024,"ok":696,"rps":115,"p99_us":5397311,"fail":0},{"conc":1024,"ok":629,"rps":104,"p99_us":4549641,"fail":0},{"conc":2048,"ok":623,"rps":102,"p99_us":3826765,"fail":0},{"conc":2048,"ok":467,"rps":77,"p99_us":4669951,"fail":0},{"conc":2048,"ok":656,"rps":109,"p99_us":5242341,"fail":0},{"conc":4096,"ok":429,"rps":71,"p99_us":3777750,"fail":0},{"conc":4096,"ok":523,"rps":87,"p99_us":5632784,"fail":0},{"conc":4096,"ok":0,"rps":0,"p99_us":null,"fail":91},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":684},{"conc":8192,"ok":548,"rps":91,"p99_us":5411558,"fail":0},{"conc":8192,"ok":12,"rps":1,"p99_us":5960300,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":2376},{"conc":16384,"ok":33,"rps":5,"p99_us":5965146,"fail":1028},{"conc":16384,"ok":551,"rps":91,"p99_us":4408916,"fail":4095}],"rungs":[{"conc":1,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":9310,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":17781,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":35805,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":118,"certified":true,"suppressed":false},"p99_us":72270,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":145952,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":335625,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":119,"certified":true,"suppressed":false},"p99_us":851275,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":106,"certified":true,"suppressed":false},"p99_us":5338973,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":112,"certified":true,"suppressed":false},"p99_us":5466955,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":110,"certified":true,"suppressed":false},"p99_us":4656402,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":104,"certified":true,"suppressed":false},"p99_us":4549641,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":102,"certified":true,"suppressed":false},"p99_us":4669951,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":71,"certified":true,"suppressed":false},"p99_us":3777750,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":1,"certified":true,"suppressed":false},"p99_us":5411558,"windows":3,"clean_windows":2},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"reverify_note":"openai>openai is a same-dialect cell: there is no translation to prove, so a request-shape check upstream cannot distinguish a translating gateway from a forwarding one","c1_note":"the c=1 percentiles are taken over 701 successful gateway round trip(s) and 267297 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":701,"direct_c1_samples":267297,"rps_per_dollar":{"value":716.9117647058823,"certified":true,"suppressed":false},"cost_per_million_usd":{"value":0.3874643874643875,"certified":true,"suppressed":false},"priced_at_bound_ms":10,"gateway_hourly_usd":0.1632},"translation_cell":{"path":{"ingress":"openai","egress":"cohere"},"source":{"kind":"matrix","sweep":"6x6-translation","build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","measured_at":"2026-11-10T17:39:41Z"},"added_latency_p50_us":{"value":7551,"certified":true,"suppressed":false},"added_latency_p99_us":{"value":8314,"certified":true,"suppressed":false},"gateway_c1_p99_us":{"value":8341,"certified":true,"suppressed":false},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false},"cpu_us_per_request":{"value":7593.984962406015,"certified":true,"suppressed":false},"rps_per_cpu_second":{"value":131.68316831683168,"certified":true,"suppressed":false},"cost_window_conc":{"value":8,"certified":true,"suppressed":false},"cost_core_utilisation":{"value":0.2502569364427439,"certified":true,"suppressed":false},"cost_window_ok":{"value":798,"certified":true,"suppressed":false},"cost_window_rps":{"value":132,"certified":true,"suppressed":false},"cost_threads":{"value":13,"certified":true,"suppressed":false},"cost_nonvol_ctxt_per_request":{"value":0.03132832080200501,"certified":true,"suppressed":false},"cost_majflt":{"value":0,"certified":true,"suppressed":false},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":2,"lower_bound":false,"samples":787},{"bound_ms":50,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":8,"lower_bound":false,"samples":787},{"bound_ms":100,"rps":{"value":131,"certified":true,"suppressed":false},"concurrency":1,"p99_us":8409,"first_disqualified_conc":16,"lower_bound":false,"samples":787},{"bound_ms":null,"rps":{"value":138,"certified":true,"suppressed":false},"concurrency":64,"p99_us":510607,"first_disqualified_conc":16384,"lower_bound":false,"samples":829}],"sweep":[{"conc":1,"ok":776,"rps":129,"p99_us":8508,"fail":0},{"conc":1,"ok":783,"rps":130,"p99_us":8417,"fail":0},{"conc":1,"ok":787,"rps":131,"p99_us":8409,"fail":0},{"conc":2,"ok":784,"rps":130,"p99_us":16280,"fail":0},{"conc":2,"ok":788,"rps":131,"p99_us":16118,"fail":0},{"conc":2,"ok":787,"rps":131,"p99_us":16403,"fail":0},{"conc":4,"ok":783,"rps":130,"p99_us":32617,"fail":0},{"conc":4,"ok":785,"rps":130,"p99_us":31863,"fail":0},{"conc":4,"ok":779,"rps":129,"p99_us":32369,"fail":0},{"conc":8,"ok":781,"rps":130,"p99_us":64987,"fail":0},{"conc":8,"ok":783,"rps":130,"p99_us":66235,"fail":0},{"conc":8,"ok":783,"rps":130,"p99_us":65825,"fail":0},{"conc":16,"ok":788,"rps":131,"p99_us":132178,"fail":0},{"conc":16,"ok":789,"rps":131,"p99_us":133406,"fail":0},{"conc":16,"ok":789,"rps":131,"p99_us":133940,"fail":0},{"conc":32,"ok":797,"rps":132,"p99_us":356272,"fail":0},{"conc":32,"ok":794,"rps":132,"p99_us":338697,"fail":0},{"conc":32,"ok":793,"rps":132,"p99_us":366565,"fail":0},{"conc":64,"ok":746,"rps":123,"p99_us":939079,"fail":0},{"conc":64,"ok":814,"rps":135,"p99_us":515904,"fail":0},{"conc":64,"ok":829,"rps":138,"p99_us":510607,"fail":0},{"conc":128,"ok":707,"rps":117,"p99_us":5259210,"fail":0},{"conc":128,"ok":765,"rps":127,"p99_us":4788836,"fail":0},{"conc":128,"ok":711,"rps":117,"p99_us":5233145,"fail":0},{"conc":256,"ok":759,"rps":126,"p99_us":5341012,"fail":0},{"conc":256,"ok":704,"rps":117,"p99_us":5445613,"fail":0},{"conc":256,"ok":641,"rps":106,"p99_us":5187951,"fail":0},{"conc":512,"ok":756,"rps":125,"p99_us":5315641,"fail":0},{"conc":512,"ok":647,"rps":107,"p99_us":4358299,"fail":0},{"conc":512,"ok":775,"rps":129,"p99_us":5397296,"fail":0},{"conc":1024,"ok":732,"rps":121,"p99_us":5112150,"fail":0},{"conc":1024,"ok":772,"rps":128,"p99_us":5311705,"fail":0},{"conc":1024,"ok":735,"rps":122,"p99_us":5176616,"fail":0},{"conc":2048,"ok":518,"rps":86,"p99_us":4408610,"fail":0},{"conc":2048,"ok":657,"rps":109,"p99_us":4953479,"fail":0},{"conc":2048,"ok":650,"rps":108,"p99_us":3595936,"fail":0},{"conc":4096,"ok":745,"rps":124,"p99_us":5259447,"fail":0},{"conc":4096,"ok":647,"rps":107,"p99_us":5474094,"fail":0},{"conc":4096,"ok":3,"rps":0.4999157392021575,"p99_us":5920550,"fail":504},{"conc":8192,"ok":209,"rps":34,"p99_us":5833359,"fail":0},{"conc":8192,"ok":126,"rps":20,"p99_us":5894128,"fail":0},{"conc":8192,"ok":0,"rps":0,"p99_us":null,"fail":0},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":3726},{"conc":16384,"ok":669,"rps":111,"p99_us":4588890,"fail":5645},{"conc":16384,"ok":0,"rps":0,"p99_us":null,"fail":1066}],"rungs":[{"conc":1,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":8417,"windows":3,"clean_windows":3},{"conc":2,"rps":{"value":131,"certified":true,"suppressed":false},"p99_us":16280,"windows":3,"clean_windows":3},{"conc":4,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":32369,"windows":3,"clean_windows":3},{"conc":8,"rps":{"value":130,"certified":true,"suppressed":false},"p99_us":65825,"windows":3,"clean_windows":3},{"conc":16,"rps":{"value":131,"certified":true,"suppressed":false},"p99_us":133406,"windows":3,"clean_windows":3},{"conc":32,"rps":{"value":132,"certified":true,"suppressed":false},"p99_us":356272,"windows":3,"clean_windows":3},{"conc":64,"rps":{"value":135,"certified":true,"suppressed":false},"p99_us":515904,"windows":3,"clean_windows":3},{"conc":128,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":5233145,"windows":3,"clean_windows":3},{"conc":256,"rps":{"value":117,"certified":true,"suppressed":false},"p99_us":5341012,"windows":3,"clean_windows":3},{"conc":512,"rps":{"value":125,"certified":true,"suppressed":false},"p99_us":5315641,"windows":3,"clean_windows":3},{"conc":1024,"rps":{"value":122,"certified":true,"suppressed":false},"p99_us":5176616,"windows":3,"clean_windows":3},{"conc":2048,"rps":{"value":108,"certified":true,"suppressed":false},"p99_us":4408610,"windows":3,"clean_windows":3},{"conc":4096,"rps":{"value":107,"certified":true,"suppressed":false},"p99_us":5259447,"windows":3,"clean_windows":2},{"conc":8192,"rps":{"value":20,"certified":true,"suppressed":false},"p99_us":5833359,"windows":3,"clean_windows":2},{"conc":16384,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"every window at c=16384 failed requests, so no clean rate was carried there"},"p99_us":null,"windows":3,"clean_windows":0}],"egress_reverified":true,"c1_note":"the c=1 percentiles are taken over 790 successful gateway round trip(s) and 267951 direct-to-mock round trip(s), each leg a single clean window with no failures","gateway_c1_samples":790,"direct_c1_samples":267951},"streaming":{"path":{"dialect":"openai"},"source":{"kind":"matrix","sweep":"6x6-stream-diagonal","build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","measured_at":"2026-11-10T17:39:41Z"},"stream_served":true,"added_ttft_p50_us":{"value":11154,"certified":true,"suppressed":false},"added_ttft_p99_us":{"value":11974,"certified":true,"suppressed":false},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21070us) came in under the mock's (21072us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing"},"added_gap_p99_us":{"value":9,"certified":true,"suppressed":false},"streams_sustained_fps":{"value":1207.9948436569616,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494},"streams_sustained":{"value":62,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494},"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"measured_at":"2026-11-10T17:39:41.000Z","lane_measured_at":{"perf":"2026-11-10T17:39:41Z","xlate":"2026-11-10T17:39:41Z","stream":"2026-11-10T17:39:41Z","memory":"2026-11-10T17:39:41Z"},"rig":{"arch":"arm64","release_url":"file:///home/ubuntu/benchmarking/rig","engine":{"commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","dirty":false},"mock":{"origin":"release","sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","asset_updated_at":null},"ugen":null,"box_qualify":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"somaxconn":65535,"tcp_max_syn_backlog":65535},"contributed_by":[],"stale":false,"engine":{"sha":"d240badd966933fa19eec0606b2d1f3945f18ad2","short":"d240bad","current":true},"history_family":"litellm-python","history_trends":{}},"current_metadata":{"stars":63241,"stars_as_of":"2026-11-14","first_commit":"2023-07-27"},"metadata_scope":"gateway.stars and stars_as_of are legacy aliases of current_metadata, a dated repository snapshot, not stars at measurement time. Run statistics were not recorded.","measurements":[{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"cohere","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":11850,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":12623,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":12651,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":12063.492063492064,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":82.89473684210526,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.24961199687176264,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":504,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":83,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":6,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.051587301587301584,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.81640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":663.33203125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":663.33203125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":81,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":83,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":12764,"first_disqualified_conc":4,"lower_bound":false,"samples":499},{"bound_ms":100,"rps":{"value":83,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":12764,"first_disqualified_conc":8,"lower_bound":false,"samples":499},{"bound_ms":null,"rps":{"value":112,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1024,"p99_us":3940819,"first_disqualified_conc":8192,"lower_bound":false,"samples":674}],"sample_counts":{"gateway_c1_samples":503,"direct_c1_samples":264219},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"cohere","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":301,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":7551,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":8314,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":8341,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":7593.984962406015,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":131.68316831683168,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2502569364427439,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":798,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":132,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.03132832080200501,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.56640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":666.06640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":666.0703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":666.0703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":666.0703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.0020496859180001844,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":301,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":131,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":8409,"first_disqualified_conc":2,"lower_bound":false,"samples":787},{"bound_ms":50,"rps":{"value":131,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":8409,"first_disqualified_conc":8,"lower_bound":false,"samples":787},{"bound_ms":100,"rps":{"value":131,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":8409,"first_disqualified_conc":16,"lower_bound":false,"samples":787},{"bound_ms":null,"rps":{"value":138,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":64,"p99_us":510607,"first_disqualified_conc":16384,"lower_bound":false,"samples":829}],"sample_counts":{"gateway_c1_samples":790,"direct_c1_samples":267951},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"cohere","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"anthropic","upstream_protocol":"cohere","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":8248,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":9054,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":9082,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":8407.202216066482,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":118.94563426688632,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2504942972604469,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":722,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":120,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.024930747922437674,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer cohere with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.1796875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":665.765625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":665.765625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":665.765625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":665.765625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":118,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":9145,"first_disqualified_conc":2,"lower_bound":false,"samples":713},{"bound_ms":50,"rps":{"value":120,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17574,"first_disqualified_conc":8,"lower_bound":false,"samples":722},{"bound_ms":100,"rps":{"value":120,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17574,"first_disqualified_conc":16,"lower_bound":false,"samples":722},{"bound_ms":null,"rps":{"value":120,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17574,"first_disqualified_conc":16384,"lower_bound":false,"samples":722}],"sample_counts":{"gateway_c1_samples":723,"direct_c1_samples":260556},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"cohere","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":304,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":9710,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":10423,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":10451,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":9886.363636363636,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":101.14942528735632,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25085857203068684,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":616,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":102,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.04707792207792208,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.70703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":667.8046875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":667.8046875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":304,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":104,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":20420,"first_disqualified_conc":8,"lower_bound":false,"samples":630},{"bound_ms":100,"rps":{"value":104,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":20420,"first_disqualified_conc":16,"lower_bound":false,"samples":630},{"bound_ms":null,"rps":{"value":106,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":64,"p99_us":647637,"first_disqualified_conc":8192,"lower_bound":false,"samples":637}],"sample_counts":{"gateway_c1_samples":615,"direct_c1_samples":263786},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:cohere","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"cohere","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":13881,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":14770,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":14798,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":14342.723004694835,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":69.72176759410802,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2507360988510164,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":426,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":70,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.06338028169014084,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":593.359375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":670.828125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":670.9921875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":670.9921875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":670.9921875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.09640675558624215,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":71,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":14828,"first_disqualified_conc":4,"lower_bound":false,"samples":427},{"bound_ms":100,"rps":{"value":71,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":14828,"first_disqualified_conc":8,"lower_bound":false,"samples":427},{"bound_ms":null,"rps":{"value":73,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":256,"p99_us":3307564,"first_disqualified_conc":8192,"lower_bound":false,"samples":439}],"sample_counts":{"gateway_c1_samples":428,"direct_c1_samples":262194},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"anthropic","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"anthropic","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"anthropic","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":13061,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":13826,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":13854,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":13414.096916299559,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":74.54844006568145,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25025039298228663,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":454,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":75,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.05506607929515418,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.65625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":667.640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":667.640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":667.640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":667.640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":76,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13878,"first_disqualified_conc":4,"lower_bound":false,"samples":458},{"bound_ms":100,"rps":{"value":76,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13878,"first_disqualified_conc":8,"lower_bound":false,"samples":458},{"bound_ms":null,"rps":{"value":76,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13878,"first_disqualified_conc":8192,"lower_bound":false,"samples":458}],"sample_counts":{"gateway_c1_samples":457,"direct_c1_samples":263938},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"anthropic","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":305,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":16324,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":17226,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":17254,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":16831.955922865014,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":59.41080196399345,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2501531621486448,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":363,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":60,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.0743801652892562,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.91796875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":671.87109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":671.87109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":305,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":60,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":33932,"first_disqualified_conc":4,"lower_bound":false,"samples":362},{"bound_ms":100,"rps":{"value":60,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":33932,"first_disqualified_conc":8,"lower_bound":false,"samples":362},{"bound_ms":null,"rps":{"value":60,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":33932,"first_disqualified_conc":4096,"lower_bound":false,"samples":362}],"sample_counts":{"gateway_c1_samples":364,"direct_c1_samples":262033},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"anthropic","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":304,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10313,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":11125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":11153,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10411.663807890223,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":96.04612850082371,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2500487063572403,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":583,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":97,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.03773584905660377,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":13428,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":14151,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":3,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":1348,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":1232.8623220270053,"certified":true,"suppressed":false,"headroom":0.20745279457185184,"rig_ceiling":5942.857142857143,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":117,"certified":true,"suppressed":false,"headroom":0.20745279457185184,"rig_ceiling":5942.857142857143,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.48828125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":667.12109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":667.12109375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":304,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":97,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":10954,"first_disqualified_conc":8,"lower_bound":false,"samples":586},{"bound_ms":100,"rps":{"value":98,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":8,"p99_us":88352,"first_disqualified_conc":16,"lower_bound":false,"samples":589},{"bound_ms":null,"rps":{"value":98,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":8,"p99_us":88352,"first_disqualified_conc":16384,"lower_bound":false,"samples":589}],"sample_counts":{"gateway_c1_samples":579,"direct_c1_samples":265135,"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"stream_rung_diagnostics":[{"conc":1,"http_status_counts":{"200":1},"stream_errors_no_response":0},{"conc":2,"http_status_counts":{"200":2},"stream_errors_no_response":0},{"conc":4,"http_status_counts":{"200":4},"stream_errors_no_response":0},{"conc":8,"http_status_counts":{"200":8},"stream_errors_no_response":0},{"conc":16,"http_status_counts":{"200":16},"stream_errors_no_response":0},{"conc":32,"http_status_counts":{"200":32},"stream_errors_no_response":0},{"conc":64,"http_status_counts":{"200":64},"stream_errors_no_response":0},{"conc":128,"http_status_counts":{"200":128},"stream_errors_no_response":0},{"conc":96,"http_status_counts":{"200":96},"stream_errors_no_response":0},{"conc":112,"http_status_counts":{"200":112},"stream_errors_no_response":0},{"conc":120,"http_status_counts":{"200":120},"stream_errors_no_response":0},{"conc":116,"http_status_counts":{"200":116},"stream_errors_no_response":0},{"conc":118,"http_status_counts":{"200":118},"stream_errors_no_response":0},{"conc":117,"http_status_counts":{"200":117},"stream_errors_no_response":0},{"conc":117,"http_status_counts":{"200":117},"stream_errors_no_response":0},{"conc":117,"http_status_counts":{"200":117},"stream_errors_no_response":0}],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:anthropic","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"anthropic","upstream_protocol":"anthropic","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":300,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10994,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":11745,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":11773,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":11322.160148975792,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":88.32236842105263,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2501294799366864,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":537,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":89,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":268,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.03910614525139665,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":14510,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":15815,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21123us) came in under the mock's (21136us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing","state":"below_resolution","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":3187.580899184528,"certified":true,"suppressed":false,"headroom":0.2460999958929231,"rig_ceiling":12952.380952380952,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":255,"certified":true,"suppressed":false,"headroom":0.2460999958929231,"rig_ceiling":12952.380952380952,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.48828125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":658.05859375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":658.05859375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":300,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":89,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":23288,"first_disqualified_conc":8,"lower_bound":false,"samples":538},{"bound_ms":100,"rps":{"value":89,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":23288,"first_disqualified_conc":16,"lower_bound":false,"samples":538},{"bound_ms":null,"rps":{"value":91,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":64,"p99_us":796228,"first_disqualified_conc":4096,"lower_bound":false,"samples":550}],"sample_counts":{"gateway_c1_samples":544,"direct_c1_samples":263642,"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"stream_rung_diagnostics":[{"conc":1,"http_status_counts":{"200":1},"stream_errors_no_response":0},{"conc":2,"http_status_counts":{"200":2},"stream_errors_no_response":0},{"conc":4,"http_status_counts":{"200":4},"stream_errors_no_response":0},{"conc":8,"http_status_counts":{"200":8},"stream_errors_no_response":0},{"conc":16,"http_status_counts":{"200":16},"stream_errors_no_response":0},{"conc":32,"http_status_counts":{"200":32},"stream_errors_no_response":0},{"conc":64,"http_status_counts":{"200":64},"stream_errors_no_response":0},{"conc":128,"http_status_counts":{"200":128},"stream_errors_no_response":0},{"conc":256,"http_status_counts":{"200":256},"stream_errors_no_response":0},{"conc":192,"http_status_counts":{"200":192},"stream_errors_no_response":0},{"conc":224,"http_status_counts":{"200":224},"stream_errors_no_response":0},{"conc":240,"http_status_counts":{"200":240},"stream_errors_no_response":0},{"conc":248,"http_status_counts":{"200":248},"stream_errors_no_response":0},{"conc":252,"http_status_counts":{"200":252},"stream_errors_no_response":0},{"conc":254,"http_status_counts":{"200":254},"stream_errors_no_response":0},{"conc":255,"http_status_counts":{"200":255},"stream_errors_no_response":0},{"conc":255,"http_status_counts":{"200":255},"stream_errors_no_response":0},{"conc":255,"http_status_counts":{"200":255},"stream_errors_no_response":0}],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"bedrock","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":301,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":17646,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":18361,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":18388,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":18244.04761904762,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":54.81239804241436,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2502884887591485,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":336,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":55,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":17,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.0744047619047619,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.97265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":689.4375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":689.4375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":689.4375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":689.4375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":64,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":301,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":18407,"first_disqualified_conc":4,"lower_bound":false,"samples":338},{"bound_ms":100,"rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":18407,"first_disqualified_conc":8,"lower_bound":false,"samples":338},{"bound_ms":null,"rps":{"value":57,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2048,"p99_us":5344748,"first_disqualified_conc":4096,"lower_bound":false,"samples":345}],"sample_counts":{"gateway_c1_samples":339,"direct_c1_samples":267586},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"bedrock","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"anthropic","upstream_protocol":"bedrock","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":305,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":16315,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":17156,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":17184,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":18047.337278106508,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":55.40983606557377,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25012322825186667,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":338,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":17,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.07396449704142012,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.72265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":687.4609375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":687.4609375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":305,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":34711,"first_disqualified_conc":4,"lower_bound":false,"samples":376},{"bound_ms":100,"rps":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":34711,"first_disqualified_conc":8,"lower_bound":false,"samples":376},{"bound_ms":null,"rps":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":64,"p99_us":1110053,"first_disqualified_conc":4096,"lower_bound":false,"samples":382}],"sample_counts":{"gateway_c1_samples":366,"direct_c1_samples":262394},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"bedrock","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":21239,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":22150,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":22177,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":22086.330935251797,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":45.276872964169385,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2506164650746462,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":278,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":46,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":17,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.07913669064748201,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.921875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":692.7734375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":692.77734375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":692.77734375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":692.77734375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":64,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.005149277000641089,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":46,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":22251,"first_disqualified_conc":4,"lower_bound":false,"samples":280},{"bound_ms":100,"rps":{"value":46,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":22251,"first_disqualified_conc":8,"lower_bound":false,"samples":280},{"bound_ms":null,"rps":{"value":46,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":22251,"first_disqualified_conc":4096,"lower_bound":false,"samples":280}],"sample_counts":{"gateway_c1_samples":281,"direct_c1_samples":267100},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"bedrock","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":303,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10032,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":10847,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":10874,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10134.003350083753,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":98.67768595041322,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.24972426540447598,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":597,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":99,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.04522613065326633,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.5078125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":671.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":671.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":303,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":101,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":20685,"first_disqualified_conc":8,"lower_bound":false,"samples":607},{"bound_ms":100,"rps":{"value":101,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":20685,"first_disqualified_conc":16,"lower_bound":false,"samples":607},{"bound_ms":null,"rps":{"value":104,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":128,"p99_us":1374776,"first_disqualified_conc":16384,"lower_bound":false,"samples":625}],"sample_counts":{"gateway_c1_samples":595,"direct_c1_samples":267885},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:bedrock","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"bedrock","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":14005,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":14745,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":14772,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":14262.295081967213,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":70.11494252873564,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.24985541302136363,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":427,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":71,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":17,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.0468384074941452,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer bedrock with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":593.1875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":689.1015625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":689.1015625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":72,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":28461,"first_disqualified_conc":4,"lower_bound":false,"samples":434},{"bound_ms":100,"rps":{"value":72,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":28461,"first_disqualified_conc":8,"lower_bound":false,"samples":434},{"bound_ms":null,"rps":{"value":72,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":28461,"first_disqualified_conc":8192,"lower_bound":false,"samples":434}],"sample_counts":{"gateway_c1_samples":427,"direct_c1_samples":272464},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"openai","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":8512,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":9522,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":9549,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":8644.793152639088,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":115.67656765676568,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.24990558455671374,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":701,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":116,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":32,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.028530670470756064,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":11154,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":11974,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"below_resolution","detail":"the gateway's own inter-frame gap at this percentile (21070us) came in under the mock's (21072us), which a proxy cannot really do - the added gap here is below what this rig can resolve against the mock's 20ms pacing","state":"below_resolution","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":9,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":1207.9948436569616,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":62,"certified":true,"suppressed":false,"headroom":0.3835870723305876,"rig_ceiling":3149.2063492063494,"state":"valid","reason":null,"section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.95703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":628.60546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":628.60546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":117,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":9350,"first_disqualified_conc":2,"lower_bound":false,"samples":708},{"bound_ms":50,"rps":{"value":119,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17670,"first_disqualified_conc":8,"lower_bound":false,"samples":718},{"bound_ms":100,"rps":{"value":119,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16,"lower_bound":false,"samples":718},{"bound_ms":null,"rps":{"value":119,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17670,"first_disqualified_conc":16384,"lower_bound":false,"samples":718}],"sample_counts":{"gateway_c1_samples":701,"direct_c1_samples":267297,"ttft_gw_samples":100,"ttft_direct_samples":100,"gap_gw_samples":63,"gap_direct_samples":63},"stream_rung_diagnostics":[{"conc":1,"http_status_counts":{"200":1},"stream_errors_no_response":0},{"conc":2,"http_status_counts":{"200":2},"stream_errors_no_response":0},{"conc":4,"http_status_counts":{"200":4},"stream_errors_no_response":0},{"conc":8,"http_status_counts":{"200":8},"stream_errors_no_response":0},{"conc":16,"http_status_counts":{"200":16},"stream_errors_no_response":0},{"conc":32,"http_status_counts":{"200":32},"stream_errors_no_response":0},{"conc":64,"http_status_counts":{"200":64},"stream_errors_no_response":0},{"conc":48,"http_status_counts":{"200":48},"stream_errors_no_response":0},{"conc":56,"http_status_counts":{"200":56},"stream_errors_no_response":0},{"conc":60,"http_status_counts":{"200":60},"stream_errors_no_response":0},{"conc":62,"http_status_counts":{"200":62},"stream_errors_no_response":0},{"conc":63,"http_status_counts":{"200":63},"stream_errors_no_response":0},{"conc":62,"http_status_counts":{"200":62},"stream_errors_no_response":0},{"conc":62,"http_status_counts":{"200":62},"stream_errors_no_response":0}],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_served","raw_state":"failed","reason":"failed","detail":"probed and answered HTTP 200, but the body is not a usable anthropic response (see body_snippet)","evidence_scope":"Recorded run evidence","probe_http_status":"200","response_excerpt":"the 2xx body is not a response of the anthropic dialect with assistant text at content[type=text].text: {\"id\":\"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo2MTk3ZDQ1OWNlMzc1OTE2NzVlMTIyMTg3ZDAwOTViNzIyNDhmMTRmYTUwNzQzMDZkMzFlMzlhODkzMTM2OWQ4O3Jlc3BvbnNlX2lkOmNoYXRjbXBsLXg=\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"gpt-4o-mini\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"output_tokens\":2},\"content\":[],\"stop_reason\":\"end_turn\",\"stop_details\":null}"},"test":{"ingress_protocol":"anthropic","upstream_protocol":"openai","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"openai","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":304,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10676,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":11606,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":11634,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10555.555555555555,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":94.73684210526315,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2503730077247171,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":576,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":95,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.059027777777777776,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.90625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":629.95703125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":629.96875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":629.96875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":629.96875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.00555168581824748,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":304,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":95,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":22974,"first_disqualified_conc":8,"lower_bound":false,"samples":572},{"bound_ms":100,"rps":{"value":95,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":22974,"first_disqualified_conc":16,"lower_bound":false,"samples":572},{"bound_ms":null,"rps":{"value":95,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":22974,"first_disqualified_conc":4096,"lower_bound":false,"samples":572}],"sample_counts":{"gateway_c1_samples":559,"direct_c1_samples":265389},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"openai","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"openai","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:openai","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"openai","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":14407,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":15293,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":15321,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":15086.41975308642,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":66.28477905073649,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25038840963155784,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":405,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":67,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.044444444444444446,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.00390625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":632.30859375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":632.31640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":632.31640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":632.31640625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.003008508378215942,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":68,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":15465,"first_disqualified_conc":4,"lower_bound":false,"samples":410},{"bound_ms":100,"rps":{"value":68,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":15465,"first_disqualified_conc":8,"lower_bound":false,"samples":410},{"bound_ms":null,"rps":{"value":68,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":15465,"first_disqualified_conc":8192,"lower_bound":false,"samples":410}],"sample_counts":{"gateway_c1_samples":412,"direct_c1_samples":263451},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"openai-responses","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"openai-responses","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"openai-responses","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":306,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":17577,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":18499,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":18527,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":18005.865102639294,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":55.537459283387626,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.250691676136018,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":341,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.08211143695014662,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.40625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":674.6484375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":674.6484375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":306,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":18534,"first_disqualified_conc":4,"lower_bound":false,"samples":337},{"bound_ms":100,"rps":{"value":56,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":18534,"first_disqualified_conc":8,"lower_bound":false,"samples":337},{"bound_ms":null,"rps":{"value":59,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2048,"p99_us":5599008,"first_disqualified_conc":4096,"lower_bound":false,"samples":360}],"sample_counts":{"gateway_c1_samples":338,"direct_c1_samples":262004},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"anthropic","upstream_protocol":"openai-responses","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":303,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":13287,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":14010,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":14038,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":13738.738738738739,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":72.78688524590164,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25008979839096346,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":444,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":73,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.07207207207207207,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.65234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":661.2265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":661.2265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":63,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":303,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":74,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13937,"first_disqualified_conc":4,"lower_bound":false,"samples":450},{"bound_ms":100,"rps":{"value":74,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13937,"first_disqualified_conc":8,"lower_bound":false,"samples":450},{"bound_ms":null,"rps":{"value":74,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":13937,"first_disqualified_conc":8192,"lower_bound":false,"samples":450}],"sample_counts":{"gateway_c1_samples":449,"direct_c1_samples":260854},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"openai-responses","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":305,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10726,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":11474,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":11502,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10799.289520426288,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":92.59868421052632,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2502330796843856,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":563,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":93,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.05328596802841918,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.7890625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":669.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":669.40234375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":305,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":92,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":11558,"first_disqualified_conc":8,"lower_bound":false,"samples":556},{"bound_ms":100,"rps":{"value":92,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":11558,"first_disqualified_conc":16,"lower_bound":false,"samples":556},{"bound_ms":null,"rps":{"value":93,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":16,"p99_us":183258,"first_disqualified_conc":16384,"lower_bound":false,"samples":559}],"sample_counts":{"gateway_c1_samples":557,"direct_c1_samples":261925},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:openai-responses","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"openai-responses","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":300,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":11554,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":12280,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":12308,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":11692.307692307691,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":85.52631578947368,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2497786700112885,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":520,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":86,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.06346153846153846,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.46484375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":627.4453125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":629.47265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":629.47265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":629.47265625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":3.0355931092621313,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":300,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":87,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":23671,"first_disqualified_conc":8,"lower_bound":false,"samples":524},{"bound_ms":100,"rps":{"value":87,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":23671,"first_disqualified_conc":16,"lower_bound":false,"samples":524},{"bound_ms":null,"rps":{"value":88,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":32,"p99_us":556619,"first_disqualified_conc":8192,"lower_bound":false,"samples":529}],"sample_counts":{"gateway_c1_samples":518,"direct_c1_samples":265921},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai-responses:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai-responses","upstream_protocol":"gemini","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":304,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10486,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":11201,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":11229,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10628.272251308901,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":94.08866995073892,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25043632937727195,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":573,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":95,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.041884816753926704,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer openai-responses with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":592.390625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":669.41015625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":669.4140625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":669.4140625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":669.4140625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":62,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":304,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":96,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21588,"first_disqualified_conc":8,"lower_bound":false,"samples":578},{"bound_ms":100,"rps":{"value":96,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21588,"first_disqualified_conc":16,"lower_bound":false,"samples":578},{"bound_ms":null,"rps":{"value":96,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21588,"first_disqualified_conc":16384,"lower_bound":false,"samples":578}],"sample_counts":{"gateway_c1_samples":570,"direct_c1_samples":265239},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:gemini:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"gemini","upstream_protocol":"gemini","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":10073,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":10889,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":10916,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":27,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":10236.486486486487,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":97.6897689768977,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.2501839420461313,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":592,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":98,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.048986486486486486,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":627.6875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":627.6875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":627.6875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":627.6875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 10ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":50,"rps":{"value":101,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21419,"first_disqualified_conc":8,"lower_bound":false,"samples":612},{"bound_ms":100,"rps":{"value":101,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21419,"first_disqualified_conc":16,"lower_bound":false,"samples":612},{"bound_ms":null,"rps":{"value":101,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":21419,"first_disqualified_conc":4096,"lower_bound":false,"samples":612}],"sample_counts":{"gateway_c1_samples":592,"direct_c1_samples":267671},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:cohere:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"cohere","upstream_protocol":"gemini","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:bedrock:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"not_declared","raw_state":"not_configurable","reason":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","detail":"Pinned v1.108.3 source b7d579fab2818968322ef96e5b8e98dc8178fafc: Chat, Responses, Messages and Gemini ingress translate to all six configured upstream wires. The per-model upstream setting selects Chat versus Responses without changing workload parameters. The default Messages-to-Chat bridge reaches the correct wire but returns empty content; it is configured as shipped, without a global bridge override, so its failure is recorded. Native Cohere and Bedrock ingress use the shipped /cohere and /bedrock provider-prefixed passthrough routes in litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py; passthrough is same-dialect only. Configured pairs that fail are recorded as failures. Evidence: results/diagnostics/refresh-configuration/litellm-python-final-config-correctness-preflight.json","evidence_scope":"Historical capability research, not a fresh probe of the tested build. Referenced versions may differ from gateway_build; do not infer current capability.","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"bedrock","upstream_protocol":"gemini","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":null,"memory_concurrency":null,"memory_load_s":null,"cost_concurrency":null,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{},"frontier":[],"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:openai:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"openai","upstream_protocol":"gemini","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":8225,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":8995,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":9023,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":8281.036834924966,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":120.75782537067545,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25021912323444456,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":733,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":122,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.0286493860845839,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":591.8359375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":665.40625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":665.40625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":665.40625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":665.40625,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":121,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":8923,"first_disqualified_conc":2,"lower_bound":false,"samples":732},{"bound_ms":50,"rps":{"value":123,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17053,"first_disqualified_conc":8,"lower_bound":false,"samples":740},{"bound_ms":100,"rps":{"value":123,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":17053,"first_disqualified_conc":16,"lower_bound":false,"samples":740},{"bound_ms":null,"rps":{"value":127,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":128,"p99_us":1100645,"first_disqualified_conc":8192,"lower_bound":false,"samples":767}],"sample_counts":{"gateway_c1_samples":725,"direct_c1_samples":265737},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"},{"schema_version":"1.0","measurement_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf:anthropic:gemini","gateway_id":"litellm-python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","run_id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","measured_at":"2026-11-10T17:39:41Z","status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"test":{"ingress_protocol":"anthropic","upstream_protocol":"gemini","upstream_transport":"http","hardware":"AWS m7g.4xlarge (Graviton3, 16 cores / 64 GB). Gateway-under-test pinned to 4 cores (the comparable basis); mock and load generator on 6 cores each so the mock never bottlenecks the streaming sweep. Ubuntu 24.04. One dedicated box per gateway.","architecture":"arm64","harness_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","harness_dirty":false,"mock_sha256":"99d01f709d8d9c1f49055aa265d5eb7475264d6b513911fe317aee74a71190f8","load_generator_sha256":null,"qualification":{"band_pct":20,"baseline_rps":96199,"baseline_samples":1,"concurrency":32,"drift_pct":2.8326697782721233,"observed_absent_detail":null,"observed_absent_reason":null,"observed_rps":98924,"outcome":"pass"},"definitions":{"memory":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran.","perf.added_latency":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","perf.cost":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else.","perf.frontier":"THROUGHPUT AT A TAIL LATENCY YOU ACCEPT. For each declared bound, the most requests/sec this cell carried while 99% of requests finished under that bound AND it failed none it accepted. Bounds: 1ms, 5ms, 10ms, 50ms, 100ms plus one reading with no latency bound at all (failures only). ONE MEASUREMENT, READ SIX WAYS: a single concurrency sweep, published as `sweep_max_proxy`, so every reading is re-derivable from the rungs rather than taken on trust. Monotone non-decreasing across the bounds BY CONSTRUCTION - relaxing a bound only adds rungs to the set a maximum is taken over - so a reading cannot exceed a looser one. ZERO FAILURES, not a tolerance: the load generator counts connections the RIG could not open separately (`rig_refused`, and those windows are discarded), so a failure reaching a reading is the gateway losing a request it had accepted. TERMINATION: the sweep climbs a doubling ladder from the floor and stops at the first concurrency where no window served cleanly, because more concurrency cannot un-fail those requests. Each reading names `first_disqualified_conc`, the lowest concurrency above it that stopped qualifying - that pair is the proof it is a boundary. When nothing above it disqualified, the sweep ran out of range instead of finding a limit, and the reading sets `lower_bound: true`: the rate is real, it is simply a floor rather than a ceiling. A window in which the gateway completed no request at all is a failed rung like any other. Only a RIG stop (a thread or connection this host could not make, or a load generator with no readable result) interrupts the sweep; its frontier is then read from the rungs below the stop, every reading sets `lower_bound: true`, and the absences name the rung and the cause. A bound no rung met publishes `bound_not_met`. The `p99_us` on a reading is the tail the winning rung ACTUALLY produced, never the bound - 4ms under a 100ms bound is not the same finding as 99ms - and `samples` is the number of completed requests it was taken over.","stream.added_ttft_and_gap":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added.","stream.streams_sustained":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"sweep_duration_s":null,"sweep_ttft_ms":null,"memory_protocol":"cold restart, 60s idle read at rest, then load at c=32 for 300s, then 60s with the load removed. `plateaued` is a verdict over the trailing 60s, taken at the END of the load: steady means drift under 1% and range under 2% of the mean. The verdict does not end the load - every cell is loaded for the full duration.","memory_concurrency":null,"memory_load_s":302,"cost_concurrency":8,"somaxconn":65535,"tcp_max_syn_backlog":65535},"metrics":{"added_latency_p50_us":{"value":8964,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"added_latency_p99_us":{"value":9686,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"gateway_c1_p99_us":{"value":9714,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"direct_c1_p99_us":{"value":28,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole."},"cpu_us_per_request":{"value":9144.144144144144,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"rps_per_cpu_second":{"value":109.35960591133005,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_conc":{"value":8,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_core_utilisation":{"value":0.25073457424131973,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_ok":{"value":666,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_window_rps":{"value":110,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_threads":{"value":13,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_nonvol_ctxt_per_request":{"value":0.03453453453453453,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"cost_majflt":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition":"WHAT A REQUEST COST, in microseconds of gateway CPU. User plus system time, summed across the gateway's whole PROCESS TREE (a gateway that forks workers keeps its CPU in the children, and a parent-only reading would report a busy gateway as idle), sampled as the DIFFERENCE across one load window and divided by the requests that window actually completed. WHY IT EXISTS BESIDE THROUGHPUT. Peak rps is a SATURATION number: once a gateway fills the cores it is pinned to, the ladder stops describing the gateway and starts describing the box, and two gateways at that wall read the same however different they are. Cost per request has no such ceiling - at saturation both serve the same rate by definition, and the one doing less work per request still reads lower. It is also the figure that maps to money, since half the CPU serves the same traffic on half the instance. MEASURED AT ONE CONCURRENCY, c=8, HELD IDENTICAL FOR EVERY GATEWAY and published as `cost_window_conc`. No single concurrency is below saturation for a field spanning double-digit to five-figure rps, so matched LOAD is impossible and matched CONCURRENCY is the honest substitute - and it is stated rather than assumed. REFUSED, NOT ESTIMATED, IN TWO CASES. A window with ANY failure publishes no cost at all: CPU spent refusing requests is real, but dividing it by only the successes would describe the failures rather than the work. And a window with major page faults means the box was SWAPPING, so what was timed is the disk - those numbers publish with `cost_majflt` non-zero and the cost itself marked a harness fault, so a reader sees why the row looks wrong instead of finding a hole. `cost_core_utilisation` is the fraction of the PINNED cores busy across that same window. It is what makes a peak interpretable: near 1.0 the gateway had filled the cores it was given and its throughput number is a real ceiling; well below it, the limit is somewhere else and the peak means something else."},"added_ttft_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p50_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"added_gap_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"streams_sustained_fps":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"streams_sustained":{"value":null,"certified":false,"suppressed":false,"reason":"untestable","detail":"the mock does not answer gemini with a native event stream, so the rig cannot pose the streaming question here","state":"untestable","section":"stream","definition":"THE MOST CONCURRENT SSE STREAMS THIS CELL CARRIES CLEANLY, and the frames/sec at that point. A concurrency holds the gate when every expected content frame arrives (ratio 1.0, i.e. no tolerance - a proxy that drops a frame has dropped a user's token), no inter-frame gap exceeds 10x the mock's own pacing interval, and under 0.1% of streams error. TERMINATION: the gate is monotone in concurrency, so the ceiling is a boundary found by bisection - a pass proven at n and a failure measured above it. `sweep_streams` carries every rung probed. A range whose top still passed publishes an absence rather than a ceiling, because the top of the range is our choice and not the gateway's answer."},"idle_rss_mib":{"value":593.02734375,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"steady_state_rss_mib":{"value":666.55078125,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"recovered_rss_mib":{"value":666.5546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_mib":{"value":666.5546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"peak_rss_hwm_mib":{"value":666.5546875,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"time_to_plateau_s":{"value":61,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"growth_rate_mib_per_min":{"value":0.0057259229747044565,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"load_s":{"value":302,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_shape":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"a settled idle window has no unsettled shape to describe","state":"not_measured","section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_static":{"value":1,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."},"idle_growth_rate_mib_per_min":{"value":0,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"memory","definition":"See `matrix.memory.protocol` and each cell's `memory.protocol`, which state the durations and both plateau thresholds formatted from the constants that ran."}},"frontier":[{"bound_ms":1,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 1ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":5,"rps":{"value":null,"certified":false,"suppressed":false,"reason":"bound_not_met","detail":"every cleanly-served rung that reported a tail had it at or above 5ms, so this gateway carried no measurable throughput under that bound","state":"bound_not_met"},"concurrency":null,"p99_us":null,"first_disqualified_conc":null,"lower_bound":false},{"bound_ms":10,"rps":{"value":111,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":1,"p99_us":9727,"first_disqualified_conc":2,"lower_bound":false,"samples":667},{"bound_ms":50,"rps":{"value":113,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":19230,"first_disqualified_conc":8,"lower_bound":false,"samples":682},{"bound_ms":100,"rps":{"value":113,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":19230,"first_disqualified_conc":16,"lower_bound":false,"samples":682},{"bound_ms":null,"rps":{"value":113,"certified":true,"suppressed":false,"state":"valid","reason":null},"concurrency":2,"p99_us":19230,"first_disqualified_conc":16384,"lower_bound":false,"samples":682}],"sample_counts":{"gateway_c1_samples":666,"direct_c1_samples":263751},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation"}],"history":[{"id":"run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","gateway_id":"litellm-python","measured_at":"2026-11-10T17:39:41Z","build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","gateway_family":"litellm-python","engine":"d240badd966933fa19eec0606b2d1f3945f18ad2","projector":"historical-matrix-v1-e1440c114a2bba3bde4a01bfa138bc6160e327576b1d93bd2dced418d56805ae","configuration_sha256":"34cf18d84e69e4f1863563157d142520c03bc9b89b830828302d46ca99d66824","disposition":"recorded","evidence_available":true,"available_metrics":["added_gap_p50_us","added_gap_p99_us","added_latency_p50_us","added_latency_p99_us","added_ttft_p50_us","added_ttft_p99_us","cost_core_utilisation","cost_majflt","cost_nonvol_ctxt_per_request","cost_threads","cost_window_conc","cost_window_ok","cost_window_rps","cpu_us_per_request","direct_c1_p99_us","gateway_c1_p99_us","growth_rate_mib_per_min","idle_growth_rate_mib_per_min","idle_rss_mib","idle_shape","idle_static","load_s","peak_rss_hwm_mib","peak_rss_mib","recovered_rss_mib","rps_per_cpu_second","shape","steady_state_rss_mib","streams_sustained","streams_sustained_fps","time_to_plateau_s"],"measurements_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/measurements","source_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json"},{"id":"run-litellm-python-f479bedd357f332ad66a2fb412beb3c3ab9dc6e00dcf7c41a019f156c375db90","sha256":"f479bedd357f332ad66a2fb412beb3c3ab9dc6e00dcf7c41a019f156c375db90","gateway_id":"litellm-python","measured_at":"2026-08-26T19:08:49Z","build":"ghcr.io/berriai/litellm:v1.98.0","gateway_family":"litellm-python","engine":"f3adbb1315b26129f5e317af5279decefb1cea8f","projector":"historical-matrix-v1-e1440c114a2bba3bde4a01bfa138bc6160e327576b1d93bd2dced418d56805ae","configuration_sha256":"9bedac31794c0a4e6767f633194212974437fdd1808eeaade1d236c3483400a4","disposition":"recorded","evidence_available":true,"available_metrics":["added_gap_p50_us","added_gap_p99_us","added_latency_p50_us","added_latency_p99_us","added_ttft_p50_us","added_ttft_p99_us","cost_core_utilisation","cost_majflt","cost_nonvol_ctxt_per_request","cost_threads","cost_window_conc","cost_window_ok","cost_window_rps","cpu_us_per_request","direct_c1_p99_us","gateway_c1_p99_us","growth_rate_mib_per_min","idle_growth_rate_mib_per_min","idle_rss_mib","idle_shape","idle_static","load_s","peak_rss_hwm_mib","peak_rss_mib","recovered_rss_mib","rps_per_cpu_second","shape","steady_state_rss_mib","streams_sustained","streams_sustained_fps","time_to_plateau_s"],"measurements_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-f479bedd357f332ad66a2fb412beb3c3ab9dc6e00dcf7c41a019f156c375db90/measurements","source_url":"https://onthebench.ai/gateways/runs/run-litellm-python-f479bedd357f332ad66a2fb412beb3c3ab9dc6e00dcf7c41a019f156c375db90","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-f479bedd357f332ad66a2fb412beb3c3ab9dc6e00dcf7c41a019f156c375db90/raw.json"}],"generated_at":"2026-11-14T11:09:24.858Z","disclosure":{"operator":"Busbar team","operator_is_entrant":true}}
