{"schema_version":"1.1","active_field":{"id":"otb-20261110T173446Z","engine_commit":"d240badd966933fa19eec0606b2d1f3945f18ad2","started_at":"2026-11-10T17:34:46Z","gateways":["agentgateway","aisix","apisix","bifrost","busbar","gomodel","helicone","higress","kong","litellm-python","litellm-rust","one-api","plano","portkey","tensorzero"]},"definitions":{"definition-1":"WHAT THE GATEWAY ADDS, at concurrency 1. The gateway leg's percentile minus the same request taken straight to the mock, one at a time, both legs in the same window. No search and no threshold: there is nothing to stop early and nothing to decide. A difference at or below what the rig can resolve publishes `below_resolution` - the best result the comparison can express - and every surface renders that apart from a never-measured hole.","definition-2":"WHAT THE GATEWAY ADDS TO A STREAM, at concurrency 1: time to the first event, and the gaps between frames inside one stream, each as the gateway leg minus the same stream taken straight to the mock. Time-to-first-EVENT rather than first token, deliberately and by the same definition on both legs: a frame budget of one is satisfied by a dialect's opening scaffolding (openai sends a role delta, anthropic a `message_start`), so the quantity is the same on both sides and the difference still isolates what the gateway added."},"dataset_id":"dataset-c7c493902ee4473d65d1886e06f885af4adac2f4f5a4ea5fc70f2efa5f230c2f","generated_at":"2026-11-14T11:09:24.858Z","published_at":"2026-11-14T11:09:23.855Z","license":"https://creativecommons.org/licenses/by/4.0/","selected_protocol_pair":{"ingress_protocol":"openai","upstream_protocol":"openai"},"results":[{"gateway_id":"agentgateway","name":"agentgateway","gateway_build":"ghcr.io/agentgateway/agentgateway:v1.7.0@sha256:4562330cd6e580bfac2de5b8905fee608dcf3fff6f1f4900744d27bdddb3b41b","measured_at":"2026-11-10T17:39:19Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":6183,"stars_as_of":"2026-11-14","first_commit":"2025-03-18"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":216,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":251,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":360,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/agentgateway","raw_result_url":"https://onthebench.ai/api/v1/runs/run-agentgateway-fa351efdcee949e977dd1d4331440512b3e913a74b878f4bd1b19a7c7ca2900c/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-agentgateway-fa351efdcee949e977dd1d4331440512b3e913a74b878f4bd1b19a7c7ca2900c","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"fa351efdcee949e977dd1d4331440512b3e913a74b878f4bd1b19a7c7ca2900c"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=agentgateway&ingress=openai&egress=openai"},{"gateway_id":"aisix","name":"AISIX (api7)","gateway_build":"ghcr.io/api7/aisix:1.9.0@sha256:dbe230bdaa4f353f7bf99d5437ab11f8b67435afabb217cf371c441ede43270a","measured_at":"2026-11-10T17:39:23Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":219,"stars_as_of":"2026-11-14","first_commit":"2026-04-17"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":146,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":174,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":366,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/aisix","raw_result_url":"https://onthebench.ai/api/v1/runs/run-aisix-ef0cff28391e0017f17c4b7c01e0171c62359468b787de48402bc3408df76915/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-aisix-ef0cff28391e0017f17c4b7c01e0171c62359468b787de48402bc3408df76915","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"ef0cff28391e0017f17c4b7c01e0171c62359468b787de48402bc3408df76915"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=aisix&ingress=openai&egress=openai"},{"gateway_id":"apisix","name":"APISIX","gateway_build":"apache/apisix:3.19.0-debian@sha256:7401ce7cfd56de1b9dcecfe7d7c7e5a681bc29222767fd28391995a0936eaa38","measured_at":"2026-11-10T17:39:33Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":17321,"stars_as_of":"2026-11-14","first_commit":"2019-04-10"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":179,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":340,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":11228,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/apisix","raw_result_url":"https://onthebench.ai/api/v1/runs/run-apisix-cc145fc7e19bd52b5c47dc4c3db32fb0511f656418b54e198d2b3cf922523d3f/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-apisix-cc145fc7e19bd52b5c47dc4c3db32fb0511f656418b54e198d2b3cf922523d3f","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"cc145fc7e19bd52b5c47dc4c3db32fb0511f656418b54e198d2b3cf922523d3f"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=apisix&ingress=openai&egress=openai"},{"gateway_id":"bifrost","name":"Bifrost","gateway_build":"maximhq/bifrost:v2.5.0@sha256:445bc49544bc64971b63581a93c6ca5716a03ccf27a58ced4c285cbafc133e78","measured_at":"2026-11-11T03:17:59Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":9550,"stars_as_of":"2026-11-14","first_commit":"2025-03-19"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":658,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":1057,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":1417,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/bifrost","raw_result_url":"https://onthebench.ai/api/v1/runs/run-bifrost-c253d4668350f1793f587661b425702d5448ca78d04ec9144589514b6064ebd7/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-bifrost-c253d4668350f1793f587661b425702d5448ca78d04ec9144589514b6064ebd7","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"c253d4668350f1793f587661b425702d5448ca78d04ec9144589514b6064ebd7"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=bifrost&ingress=openai&egress=openai"},{"gateway_id":"busbar","name":"Busbar","gateway_build":"getbusbar/busbar:1.5.5@sha256:b4cfbc7431946204aef5b1f627616e33c654a4b1e12a13758b62a9e6f71501d3","measured_at":"2026-11-10T17:39:31Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":221,"stars_as_of":"2026-11-14","first_commit":"2026-05-29"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":78,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":89,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":254,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/busbar","raw_result_url":"https://onthebench.ai/api/v1/runs/run-busbar-16d2da7bc28375e55402b153f0f49b1550d5556f6ebb71eda60b8c3c47ad8d5f/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-busbar-16d2da7bc28375e55402b153f0f49b1550d5556f6ebb71eda60b8c3c47ad8d5f","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"16d2da7bc28375e55402b153f0f49b1550d5556f6ebb71eda60b8c3c47ad8d5f"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=busbar&ingress=openai&egress=openai"},{"gateway_id":"gomodel","name":"GoModel","gateway_build":"enterpilot/gomodel:0.2.3@sha256:c50627e093ff88bff311e582a43d7de14e0be29d52cb333ecb3191eac7874f05","measured_at":"2026-11-10T17:39:52Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":1321,"stars_as_of":"2026-11-14","first_commit":"2025-12-13"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":343,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":565,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":601,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/gomodel","raw_result_url":"https://onthebench.ai/api/v1/runs/run-gomodel-f6e038780b29f86549f840f902ac96740154f0f0ef2a66a3fd9dcd4bc3b83bad/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-gomodel-f6e038780b29f86549f840f902ac96740154f0f0ef2a66a3fd9dcd4bc3b83bad","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"f6e038780b29f86549f840f902ac96740154f0f0ef2a66a3fd9dcd4bc3b83bad"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=gomodel&ingress=openai&egress=openai"},{"gateway_id":"helicone","name":"Helicone","gateway_build":"target/release/ai-gateway","measured_at":"2026-11-10T17:39:37Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":643,"stars_as_of":"2026-11-14","first_commit":"2025-04-01"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":256,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":274,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":493,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/helicone","raw_result_url":"https://onthebench.ai/api/v1/runs/run-helicone-4a55ff2dcc509c5e03f85bd92e36b949ac410653128ccc77d60019bb3aaf2558/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-helicone-4a55ff2dcc509c5e03f85bd92e36b949ac410653128ccc77d60019bb3aaf2558","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"4a55ff2dcc509c5e03f85bd92e36b949ac410653128ccc77d60019bb3aaf2558"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=helicone&ingress=openai&egress=openai"},{"gateway_id":"higress","name":"Higress","gateway_build":"higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one:2.2.5@sha256:9846006cb3cf2dafa942deab0e1d004ed30e219fbc9580438f9e3c58e7096422","measured_at":"2026-11-10T17:39:42Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":9764,"stars_as_of":"2026-11-14","first_commit":"2022-10-27"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":718,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":943,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":1103,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/higress","raw_result_url":"https://onthebench.ai/api/v1/runs/run-higress-558a1502d4a7ea1f2b74c745739723084c953a3ae5dcf16760a2694a92c21270/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-higress-558a1502d4a7ea1f2b74c745739723084c953a3ae5dcf16760a2694a92c21270","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"558a1502d4a7ea1f2b74c745739723084c953a3ae5dcf16760a2694a92c21270"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=higress&ingress=openai&egress=openai"},{"gateway_id":"kong","name":"Kong","gateway_build":"kong:3.9.4@sha256:2d572848be6fc58b73ba75edc84f028837bb7be998b3a8b8117fdb44a6892220","measured_at":"2026-11-10T17:39:46Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":44378,"stars_as_of":"2026-11-14","first_commit":"2014-11-17"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":213,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":410,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":106433,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/kong","raw_result_url":"https://onthebench.ai/api/v1/runs/run-kong-d4b6d413ea59fd281dacf70be5988600ace51a15e8665c6c7dedcc87dd2cd2fb/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-kong-d4b6d413ea59fd281dacf70be5988600ace51a15e8665c6c7dedcc87dd2cd2fb","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"d4b6d413ea59fd281dacf70be5988600ace51a15e8665c6c7dedcc87dd2cd2fb"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=kong&ingress=openai&egress=openai"},{"gateway_id":"litellm-python","name":"LiteLLM · Python","gateway_build":"ghcr.io/berriai/litellm:v1.108.3@sha256:e2bce769eb29b100d53ce90cbf2d6bba00dce82cb894c79d092ef6377523d7e5","measured_at":"2026-11-10T17:39:41Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":63241,"stars_as_of":"2026-11-14","first_commit":"2023-07-27"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":8512,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":9522,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":11974,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-python","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-python-10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"10bb57507b26d310b59f12956034785b615cd5338cdaf5cee0e587c350c53acf"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=litellm-python&ingress=openai&egress=openai"},{"gateway_id":"litellm-rust","name":"LiteLLM · Rust","gateway_build":"src/litellm-rust/target/release/litellm-gateway","measured_at":"2026-11-10T17:39:51Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":63241,"stars_as_of":"2026-11-14","first_commit":"2023-07-27"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":249,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":279,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":null,"certified":false,"suppressed":false,"reason":"not_measured","detail":"no stream frame arrived from the gateway: it answered HTTP 501, not 2xx, so what it sent is an error response rather than a stream to time","state":"not_measured","section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/litellm-rust","raw_result_url":"https://onthebench.ai/api/v1/runs/run-litellm-rust-69d8e6ed325740d9da7f0aaf31b815418c4801eb6549b2e2c68f768a92af3ad4/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-litellm-rust-69d8e6ed325740d9da7f0aaf31b815418c4801eb6549b2e2c68f768a92af3ad4","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"69d8e6ed325740d9da7f0aaf31b815418c4801eb6549b2e2c68f768a92af3ad4"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=litellm-rust&ingress=openai&egress=openai"},{"gateway_id":"one-api","name":"One-API","gateway_build":"justsong/one-api:v0.6.10@sha256:af6b361630bdf408b86fce7aa8fc131b37ea5177ae866d121ba7c002e773e477","measured_at":"2026-11-10T17:39:58Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":37431,"stars_as_of":"2026-11-14","first_commit":"2023-04-22"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":34017,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":589609,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":770,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/one-api","raw_result_url":"https://onthebench.ai/api/v1/runs/run-one-api-c9427cc70e392f48beb8e68d908f8924a7330748ac291dab1cd902b3f14c542f/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-one-api-c9427cc70e392f48beb8e68d908f8924a7330748ac291dab1cd902b3f14c542f","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"c9427cc70e392f48beb8e68d908f8924a7330748ac291dab1cd902b3f14c542f"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=one-api&ingress=openai&egress=openai"},{"gateway_id":"plano","name":"Plano","gateway_build":"katanemo/plano:0.4.39@sha256:35a84cb873ee5f16cded33eeda72736f90798962b7489d58650c0db7b546e9ac","measured_at":"2026-11-10T17:40:07Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":7117,"stars_as_of":"2026-11-14","first_commit":"2024-07-09"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":198988,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":205966,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":170494,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/plano","raw_result_url":"https://onthebench.ai/api/v1/runs/run-plano-0a4ce97da36f11e7c407eef9280f69688b7ea1f007616709f0b914993cb8ba8e/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-plano-0a4ce97da36f11e7c407eef9280f69688b7ea1f007616709f0b914993cb8ba8e","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"0a4ce97da36f11e7c407eef9280f69688b7ea1f007616709f0b914993cb8ba8e"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=plano&ingress=openai&egress=openai"},{"gateway_id":"portkey","name":"Portkey","gateway_build":"portkeyai/gateway:1.15.2@sha256:9065e15701d5f90ae1728c57369f8cec137d8c7dffcd027a402a67e64f27b9d3","measured_at":"2026-11-10T17:39:55Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":13377,"stars_as_of":"2026-11-14","first_commit":"2023-08-23"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":1196,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":3902,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":29752,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/portkey","raw_result_url":"https://onthebench.ai/api/v1/runs/run-portkey-670dce412ba19689b30f84f64cb8b9c31fc07125970bd210617247c56c23bfcc/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-portkey-670dce412ba19689b30f84f64cb8b9c31fc07125970bd210617247c56c23bfcc","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"670dce412ba19689b30f84f64cb8b9c31fc07125970bd210617247c56c23bfcc"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=portkey&ingress=openai&egress=openai"},{"gateway_id":"tensorzero","name":"TensorZero","gateway_build":"tensorzero/gateway:2026.6.0@sha256:9133135ccdaa23c0fffc2965f20b2a89b077ea3ad9a86a6f15b2a641bff67c89","measured_at":"2026-11-10T17:40:04Z","published_at":"2026-11-14T11:09:23.855Z","generated_at":"2026-11-14T11:09:24.858Z","age_days":3,"age_as_of":"2026-11-14T11:09:24.858Z","age":"3 days old at collection generation","current_metadata":{"stars":11701,"stars_as_of":"2026-11-14","first_commit":"2024-07-16"},"status":{"state":"served","raw_state":true,"reason":null,"detail":null,"evidence_scope":"Recorded run evidence","probe_http_status":null,"response_excerpt":null},"metrics":{"added_latency_p50_us":{"value":40976,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_latency_p99_us":{"value":41966,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"perf","definition_id":"definition-1"},"added_ttft_p99_us":{"value":746,"certified":true,"suppressed":false,"state":"valid","reason":null,"section":"stream","definition_id":"definition-2"}},"provenance":{"source_url":"https://onthebench.ai/gateways/tensorzero","raw_result_url":"https://onthebench.ai/api/v1/runs/run-tensorzero-e1970318de8934076256f5f37dbf5d7fb6019f1dd5bb339bd959ac71a0a1a8e1/raw.json","run_url":"https://onthebench.ai/gateways/runs/run-tensorzero-e1970318de8934076256f5f37dbf5d7fb6019f1dd5bb339bd959ac71a0a1a8e1","methodology_url":"https://onthebench.ai/gateways/method","raw_sha256":"e1970318de8934076256f5f37dbf5d7fb6019f1dd5bb339bd959ac71a0a1a8e1"},"evidence_url":"https://onthebench.ai/api/v1/measurements?gateway=tensorzero&ingress=openai&egress=openai"}],"comparison_policy":{"id":"matched-cell-v1","description":"Ratios require an identical protocol pair, hardware, architecture, metric definition, clean harness commit, recorded mock digest, and metric-specific workload. Missing evidence never matches missing evidence. These are observed gateway measurements against a local mock, not a claim about model latency or statistical significance."},"disclosure":{"operator":"Busbar team","operator_is_entrant":true},"interpretation":"Selected protocol cells, not a ranking. Check /api/v1/compare before calculating ratios. Null and below-resolution values are not zero. Repository statistics are dated metadata, not measurements. published_at is the recorded release timestamp, not the first publication of historical runs; null means unrecorded. generated_at is collection build time. Resolve metric definition_id through definitions."}
