{ "caveats": [ "Finite lab cohort; configured 262144 context differs from measured inputs. Aggregate uses first-output to final-completion window. Whole-host memory sampled during speed only; missing counters remain unassessed. Broader coding quality and delivery acceptance unassessed." ], "engine": { "name": "NVIDIA NGC vLLM", "recipe": "plain", "version": "0.29.0+5013de39.dev" }, "format": "olir/0.1", "history": { "corrections": [], "supersedes": [] }, "id": "e51d5cbb032741a59bd7cde5", "measured_at": "2026-10-07T05:50:54.427704+00:00", "metrics": [ { "basis": null, "n": null, "name": "wall_seconds", "scope": "machine_total", "source_field": "wall_seconds", "statistic": "single", "unit": "s", "value": 201.55434799194336 }, { "basis": null, "n": 20, "name": "ttft_median_s", "scope": "per_person", "source_field": "ttft_median_s", "statistic": "median", "unit": "s", "value": 113.83728480502032 }, { "basis": null, "n": null, "name": "needle_total", "scope": "machine_total", "source_field": "needle_total", "statistic": "single", "unit": "count", "value": 60 }, { "basis": null, "n": 20, "name": "ttft_max_s", "scope": "per_person", "source_field": "ttft_max_s", "statistic": "max", "unit": "s", "value": 161.927070937003 }, { "basis": null, "n": null, "name": "requests_ok", "scope": "machine_total", "source_field": "requests_ok", "statistic": "single", "unit": "count", "value": 20 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "machine_total", "source_field": "needle_correct", "statistic": "single", "unit": "count", "value": 39 }, { "basis": null, "n": null, "name": "preemptions", "scope": "machine_total", "source_field": "preemptions", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "decode_overlap", "scope": "machine_total", "source_field": "decode_overlap", "statistic": "single", "unit": "count", "value": 20 }, { "basis": null, "n": null, "name": "aggregate_concurrency", "scope": "machine_total", "source_field": "aggregate_concurrency", "statistic": "single", "unit": "count", "value": 20 }, { "basis": null, "n": null, "name": "requests_total", "scope": "machine_total", "source_field": "requests_total", "statistic": "single", "unit": "count", "value": 20 }, { "basis": null, "n": null, "name": "correctness_probes_passed", "scope": "machine_total", "source_field": "correctness_probes_passed", "statistic": "single", "unit": "count", "value": 13 }, { "basis": null, "n": null, "name": "aggregate_decode_tps", "scope": "machine_total", "source_field": "aggregate_tps", "statistic": "single", "unit": "tok/s", "value": 71.6468224183778 }, { "basis": null, "n": null, "name": "aggregate_e2e_tps", "scope": "machine_total", "source_field": "aggregate_end_to_end_tps", "statistic": "single", "unit": "tok/s", "value": 50.80515554251064 }, { "basis": null, "n": null, "name": "correctness_probes_total", "scope": "machine_total", "source_field": "correctness_probes_total", "statistic": "single", "unit": "count", "value": 20 }, { "basis": null, "n": 20, "name": "decode_tps", "scope": "per_person", "source_field": "per_request_decode_tps", "statistic": "mean", "unit": "tok/s", "value": 6.892859712043292 }, { "basis": null, "n": null, "name": "forced_output_tokens", "scope": "machine_total", "source_field": "forced_output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "peak_host_unavailable_gib", "scope": "machine_total", "source_field": "peak_host_unavailable_gib", "statistic": "max", "unit": "GiB", "value": 94.15425109863281 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[0].ttft_s", "statistic": "single", "unit": "s", "value": 148.91227467701538 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[0].total_s", "statistic": "single", "unit": "s", "value": 201.11776147800265 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[0].decode_tps", "statistic": "single", "unit": "tok/s", "value": 9.788243177350017 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[0].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[0].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[0].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[0].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[0].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[1].ttft_s", "statistic": "single", "unit": "s", "value": 93.12094456399791 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[1].total_s", "statistic": "single", "unit": "s", "value": 199.43900388700422 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[1].decode_tps", "statistic": "single", "unit": "tok/s", "value": 4.806333028027948 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[1].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[1].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40101 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[1].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[1].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[1].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[2].ttft_s", "statistic": "single", "unit": "s", "value": 62.210124173027 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[2].total_s", "statistic": "single", "unit": "s", "value": 198.19672669901047 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[2].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.7577231176311012 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[2].user_number", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[2].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40099 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[2].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[2].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[2].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[3].ttft_s", "statistic": "single", "unit": "s", "value": 93.12125973199727 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[3].total_s", "statistic": "single", "unit": "s", "value": 199.44018414401216 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[3].decode_tps", "statistic": "single", "unit": "tok/s", "value": 4.806293920165476 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[3].user_number", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[3].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40103 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[3].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[3].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[3].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[4].ttft_s", "statistic": "single", "unit": "s", "value": 119.838628721016 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[4].total_s", "statistic": "single", "unit": "s", "value": 200.28996130701853 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[4].decode_tps", "statistic": "single", "unit": "tok/s", "value": 6.351666076553059 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[4].user_number", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[4].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[4].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[4].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[4].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[5].ttft_s", "statistic": "single", "unit": "s", "value": 126.1290391900111 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[5].total_s", "statistic": "single", "unit": "s", "value": 200.47818547097268 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[5].decode_tps", "statistic": "single", "unit": "tok/s", "value": 6.872977371777175 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[5].user_number", "statistic": "single", "unit": "count", "value": 5 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[5].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[5].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[5].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[5].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[6].ttft_s", "statistic": "single", "unit": "s", "value": 60.640390462009236 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[6].total_s", "statistic": "single", "unit": "s", "value": 198.11452466598712 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[6].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.7170628712002025 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[6].user_number", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[6].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[6].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[6].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[6].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[7].ttft_s", "statistic": "single", "unit": "s", "value": 62.21007188403746 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[7].total_s", "statistic": "single", "unit": "s", "value": 198.19897111598402 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[7].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.757659653737058 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[7].user_number", "statistic": "single", "unit": "count", "value": 7 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[7].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40103 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[7].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[7].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[7].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[8].ttft_s", "statistic": "single", "unit": "s", "value": 107.83594088902464 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[8].total_s", "statistic": "single", "unit": "s", "value": 199.93539294903167 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[8].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.548350056057446 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[8].user_number", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[8].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40104 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[8].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[8].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[8].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[9].ttft_s", "statistic": "single", "unit": "s", "value": 161.927070937003 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[9].total_s", "statistic": "single", "unit": "s", "value": 201.55384508304996 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[9].decode_tps", "statistic": "single", "unit": "tok/s", "value": 12.895321686208359 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[9].user_number", "statistic": "single", "unit": "count", "value": 9 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[9].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40099 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[9].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[9].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[9].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[10].ttft_s", "statistic": "single", "unit": "s", "value": 137.90526522602886 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[10].total_s", "statistic": "single", "unit": "s", "value": 200.80344489502022 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[10].decode_tps", "statistic": "single", "unit": "tok/s", "value": 8.1242414754957 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[10].user_number", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[10].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[10].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[10].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[10].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[11].ttft_s", "statistic": "single", "unit": "s", "value": 161.5899823899963 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[11].total_s", "statistic": "single", "unit": "s", "value": 201.52797863003798 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[11].decode_tps", "statistic": "single", "unit": "tok/s", "value": 12.794833194151922 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[11].user_number", "statistic": "single", "unit": "count", "value": 11 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[11].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40106 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[11].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[11].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[11].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[12].ttft_s", "statistic": "single", "unit": "s", "value": 58.90822813601699 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[12].total_s", "statistic": "single", "unit": "s", "value": 198.0308507910231 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[12].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.6730187387795943 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[12].user_number", "statistic": "single", "unit": "count", "value": 12 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[12].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[12].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[12].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[12].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[13].ttft_s", "statistic": "single", "unit": "s", "value": 158.87563084997237 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[13].total_s", "statistic": "single", "unit": "s", "value": 201.4258340059896 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[13].decode_tps", "statistic": "single", "unit": "tok/s", "value": 12.009343366148817 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[13].user_number", "statistic": "single", "unit": "count", "value": 13 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[13].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40104 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[13].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[13].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[13].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[14].ttft_s", "statistic": "single", "unit": "s", "value": 145.74805730499793 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[14].total_s", "statistic": "single", "unit": "s", "value": 201.01954388001468 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[14].decode_tps", "statistic": "single", "unit": "tok/s", "value": 9.245273316585209 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[14].user_number", "statistic": "single", "unit": "count", "value": 14 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[14].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[14].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[14].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[14].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[15].ttft_s", "statistic": "single", "unit": "s", "value": 153.568589943985 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[15].total_s", "statistic": "single", "unit": "s", "value": 201.25991954101482 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[15].decode_tps", "statistic": "single", "unit": "tok/s", "value": 10.714735871650442 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[15].user_number", "statistic": "single", "unit": "count", "value": 15 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[15].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[15].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[15].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[15].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[16].ttft_s", "statistic": "single", "unit": "s", "value": 91.52825919998577 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[16].total_s", "statistic": "single", "unit": "s", "value": 199.37849916698178 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[16].decode_tps", "statistic": "single", "unit": "tok/s", "value": 4.7380515811218835 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[16].user_number", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[16].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40099 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[16].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[16].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[16].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[17].ttft_s", "statistic": "single", "unit": "s", "value": 124.6268373390194 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[17].total_s", "statistic": "single", "unit": "s", "value": 200.43525092600612 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[17].decode_tps", "statistic": "single", "unit": "tok/s", "value": 6.740676605950218 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[17].user_number", "statistic": "single", "unit": "count", "value": 17 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[17].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40097 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[17].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[17].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[17].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[18].ttft_s", "statistic": "single", "unit": "s", "value": 62.209566475998145 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[18].total_s", "statistic": "single", "unit": "s", "value": 198.19883554801345 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[18].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.757649434304936 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[18].user_number", "statistic": "single", "unit": "count", "value": 18 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[18].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[18].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[18].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[18].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[19].ttft_s", "statistic": "single", "unit": "s", "value": 62.21057442901656 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[19].total_s", "statistic": "single", "unit": "s", "value": 198.19657693902263 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[19].decode_tps", "statistic": "single", "unit": "tok/s", "value": 3.75773969796928 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[19].user_number", "statistic": "single", "unit": "count", "value": 19 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[19].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[19].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[19].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[19].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": 20, "name": "decode_tps", "scope": "per_person", "source_field": "per_person_decode_median_tps", "statistic": "median", "unit": "tok/s", "value": 5.950008066305252 } ], "model": { "artifact_sha256": [ "648fe83171485f4be03ea58d86089e088c94c2a3dc88922233f9002f30d8bc8b", "8303cef7d375021d69dcafc60d3fad5f8186995b18b0db5e2a609aadeb8a4f51", "cb5c5978721a1e87406ae68f1cd5a855326e40ff8bc13832644c392115336987" ], "publisher_repo": "ornith-ai/Ornith-1.5-35B-A3B", "publisher_revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1", "variant": "NVFP4 variant", "weights_dtype": "nvfp4", "weights_format": "safetensors" }, "outcome": { "gates": [ { "met": true, "name": "requests", "observed": "20/20" }, { "met": false, "name": "hidden-facts", "observed": "39/60" }, { "met": false, "name": "correctness-probes", "observed": "13/20" } ], "reasons": [ "39/60 Hidden facts found.", "13/20 probes passed." ], "status": "fail" }, "publication_group": "pre-standard", "published": null, "publisher": { "name": "Skilak Keep" }, "quality": [ { "passed": 39, "skipped": null, "suite": "hidden-facts", "total": 60, "version": "1" }, { "passed": 13, "skipped": null, "suite": "correctness-probes", "total": 20, "version": "1" } ], "record_date": "2026-10-07", "reproduce": { "driver": null, "fixture_sha256": "e2097959b1ca0142f2b93edc3ec411e474c0a6622800dd247af7a71bd9c39e83", "harness": { "name": "keep-team-load-v1", "version": "1" }, "harness_sha256": null, "os": null, "workload_name": null, "workload_version": null }, "requests": [ { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 9.788243177350017, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 201.11776147800265, "ttft_s": 148.91227467701538, "user_number": 0 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 4.806333028027948, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40101, "total_s": 199.43900388700422, "ttft_s": 93.12094456399791, "user_number": 1 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 3.7577231176311012, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40099, "total_s": 198.19672669901047, "ttft_s": 62.210124173027, "user_number": 2 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 4.806293920165476, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40103, "total_s": 199.44018414401216, "ttft_s": 93.12125973199727, "user_number": 3 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 6.351666076553059, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 200.28996130701853, "ttft_s": 119.838628721016, "user_number": 4 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 6.872977371777175, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 200.47818547097268, "ttft_s": 126.1290391900111, "user_number": 5 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 3.7170628712002025, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 198.11452466598712, "ttft_s": 60.640390462009236, "user_number": 6 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 3.757659653737058, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40103, "total_s": 198.19897111598402, "ttft_s": 62.21007188403746, "user_number": 7 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 5.548350056057446, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40104, "total_s": 199.93539294903167, "ttft_s": 107.83594088902464, "user_number": 8 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 12.895321686208359, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40099, "total_s": 201.55384508304996, "ttft_s": 161.927070937003, "user_number": 9 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 8.1242414754957, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 200.80344489502022, "ttft_s": 137.90526522602886, "user_number": 10 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 12.794833194151922, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40106, "total_s": 201.52797863003798, "ttft_s": 161.5899823899963, "user_number": 11 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 3.6730187387795943, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 198.0308507910231, "ttft_s": 58.90822813601699, "user_number": 12 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 12.009343366148817, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40104, "total_s": 201.4258340059896, "ttft_s": 158.87563084997237, "user_number": 13 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 9.245273316585209, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 201.01954388001468, "ttft_s": 145.74805730499793, "user_number": 14 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 10.714735871650442, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 201.25991954101482, "ttft_s": 153.568589943985, "user_number": 15 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 4.7380515811218835, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40099, "total_s": 199.37849916698178, "ttft_s": 91.52825919998577, "user_number": 16 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 6.740676605950218, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40097, "total_s": 200.43525092600612, "ttft_s": 124.6268373390194, "user_number": 17 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 3.757649434304936, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 198.19883554801345, "ttft_s": 62.209566475998145, "user_number": 18 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 3.75773969796928, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 198.19657693902263, "ttft_s": 62.21057442901656, "user_number": 19 } ], "scenario": { "id": "team-load", "version": "1" }, "settings": { "concurrency": 20, "configured_context_tokens": 262144, "context_tokens": 262144, "kv_dtype": "fp8", "max_output_tokens": 512, "speculative_decoding": "plain", "thinking": false }, "setup_complete": false, "setup_id": "a167e4bafd3fd96d0b47cf9d6645473d", "standard": "pre-standard", "system": { "driver": null, "hardware": "NVIDIA GB10, 128 GB unified memory", "os": null }, "workload": { "cache": null, "content_tokens": null, "document": null, "input_tokens": { "max": 40106, "median": 40102.5, "min": 40097 }, "output_tokens": 512, "repetitions": null, "users": 20, "warmup": null } }