{ "caveats": [ "Finite lab cohort; configured 262144 context differs from measured inputs. Aggregate uses first-output to final-completion window. Whole-host memory sampled during speed only; missing counters remain unassessed. Broader coding quality and delivery acceptance unassessed." ], "engine": { "name": "NVIDIA NGC vLLM", "recipe": "plain", "version": "0.29.0+5013de39.dev" }, "format": "olir/0.1", "history": { "corrections": [], "supersedes": [] }, "id": "024e2d0373e84abfb3875bf4", "measured_at": "2026-10-07T05:44:16.608851+00:00", "metrics": [ { "basis": null, "n": null, "name": "wall_seconds", "scope": "machine_total", "source_field": "wall_seconds", "statistic": "single", "unit": "s", "value": 162.72283291816711 }, { "basis": null, "n": 16, "name": "ttft_median_s", "scope": "per_person", "source_field": "ttft_median_s", "statistic": "median", "unit": "s", "value": 93.55894066250767 }, { "basis": null, "n": null, "name": "needle_total", "scope": "machine_total", "source_field": "needle_total", "statistic": "single", "unit": "count", "value": 48 }, { "basis": null, "n": 16, "name": "ttft_max_s", "scope": "per_person", "source_field": "ttft_max_s", "statistic": "max", "unit": "s", "value": 129.4477982899989 }, { "basis": null, "n": null, "name": "requests_ok", "scope": "machine_total", "source_field": "requests_ok", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "machine_total", "source_field": "needle_correct", "statistic": "single", "unit": "count", "value": 39 }, { "basis": null, "n": null, "name": "preemptions", "scope": "machine_total", "source_field": "preemptions", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "decode_overlap", "scope": "machine_total", "source_field": "decode_overlap", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "aggregate_concurrency", "scope": "machine_total", "source_field": "aggregate_concurrency", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "requests_total", "scope": "machine_total", "source_field": "requests_total", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "correctness_probes_passed", "scope": "machine_total", "source_field": "correctness_probes_passed", "statistic": "single", "unit": "count", "value": 13 }, { "basis": null, "n": null, "name": "aggregate_decode_tps", "scope": "machine_total", "source_field": "aggregate_tps", "statistic": "single", "unit": "tok/s", "value": 79.25596274786676 }, { "basis": null, "n": null, "name": "aggregate_e2e_tps", "scope": "machine_total", "source_field": "aggregate_end_to_end_tps", "statistic": "single", "unit": "tok/s", "value": 50.34327299426833 }, { "basis": null, "n": null, "name": "correctness_probes_total", "scope": "machine_total", "source_field": "correctness_probes_total", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": 16, "name": "decode_tps", "scope": "per_person", "source_field": "per_request_decode_tps", "statistic": "mean", "unit": "tok/s", "value": 8.85223287938194 }, { "basis": null, "n": null, "name": "forced_output_tokens", "scope": "machine_total", "source_field": "forced_output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "peak_host_unavailable_gib", "scope": "machine_total", "source_field": "peak_host_unavailable_gib", "statistic": "max", "unit": "GiB", "value": 94.5468864440918 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[0].ttft_s", "statistic": "single", "unit": "s", "value": 119.6756575719919 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[0].total_s", "statistic": "single", "unit": "s", "value": 162.35709048603894 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[0].decode_tps", "statistic": "single", "unit": "tok/s", "value": 11.972419038251711 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[0].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[0].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[0].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[0].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[0].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[1].ttft_s", "statistic": "single", "unit": "s", "value": 62.84592934302054 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[1].total_s", "statistic": "single", "unit": "s", "value": 160.66790649900213 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[1].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.223775013105565 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[1].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[1].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[1].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[1].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[1].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[2].ttft_s", "statistic": "single", "unit": "s", "value": 62.84495807701023 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[2].total_s", "statistic": "single", "unit": "s", "value": 160.66760278696893 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[2].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.223739365410741 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[2].user_number", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[2].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40103 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[2].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[2].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[2].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[3].ttft_s", "statistic": "single", "unit": "s", "value": 115.75592758500716 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[3].total_s", "statistic": "single", "unit": "s", "value": 162.2341767050093 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[3].decode_tps", "statistic": "single", "unit": "tok/s", "value": 10.994390057178135 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[3].user_number", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[3].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[3].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[3].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[3].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[4].ttft_s", "statistic": "single", "unit": "s", "value": 125.59252984198974 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[4].total_s", "statistic": "single", "unit": "s", "value": 162.55665049800882 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[4].decode_tps", "statistic": "single", "unit": "tok/s", "value": 13.824216319259065 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[4].user_number", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[4].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40107 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[4].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[4].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[4].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[5].ttft_s", "statistic": "single", "unit": "s", "value": 106.4212747340207 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[5].total_s", "statistic": "single", "unit": "s", "value": 161.97364485898288 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[5].decode_tps", "statistic": "single", "unit": "tok/s", "value": 9.198527422872003 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[5].user_number", "statistic": "single", "unit": "count", "value": 5 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[5].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40102 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[5].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[5].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[5].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[6].ttft_s", "statistic": "single", "unit": "s", "value": 62.8436558979447 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[6].total_s", "statistic": "single", "unit": "s", "value": 160.66437300498364 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[6].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.22384230163479 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[6].user_number", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[6].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[6].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[6].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[6].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[7].ttft_s", "statistic": "single", "unit": "s", "value": 93.55851541401353 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[7].total_s", "statistic": "single", "unit": "s", "value": 161.61167190701235 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[7].decode_tps", "statistic": "single", "unit": "tok/s", "value": 7.508836126544265 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[7].user_number", "statistic": "single", "unit": "count", "value": 7 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[7].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40101 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[7].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[7].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[7].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[8].ttft_s", "statistic": "single", "unit": "s", "value": 118.84896525798831 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[8].total_s", "statistic": "single", "unit": "s", "value": 162.33319285995094 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[8].decode_tps", "statistic": "single", "unit": "tok/s", "value": 11.751387300183673 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[8].user_number", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[8].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40106 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[8].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[8].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[8].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[9].ttft_s", "statistic": "single", "unit": "s", "value": 128.74579243297921 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[9].total_s", "statistic": "single", "unit": "s", "value": 162.67787008499727 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[9].decode_tps", "statistic": "single", "unit": "tok/s", "value": 15.059496363306511 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[9].user_number", "statistic": "single", "unit": "count", "value": 9 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[9].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40101 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[9].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[9].needle_correct", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[9].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[10].ttft_s", "statistic": "single", "unit": "s", "value": 93.5593659110018 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[10].total_s", "statistic": "single", "unit": "s", "value": 161.60899732500548 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[10].decode_tps", "statistic": "single", "unit": "tok/s", "value": 7.509225096182421 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[10].user_number", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[10].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40104 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[10].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[10].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[10].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[11].ttft_s", "statistic": "single", "unit": "s", "value": 62.844671947997995 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[11].total_s", "statistic": "single", "unit": "s", "value": 160.66355964302784 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[11].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.22393999810288 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[11].user_number", "statistic": "single", "unit": "count", "value": 11 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[11].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40106 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[11].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[11].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[11].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[12].ttft_s", "statistic": "single", "unit": "s", "value": 59.561850305995904 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[12].total_s", "statistic": "single", "unit": "s", "value": 160.53207027696772 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[12].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.060898155385902 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[12].user_number", "statistic": "single", "unit": "count", "value": 12 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[12].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40104 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[12].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[12].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[12].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[13].ttft_s", "statistic": "single", "unit": "s", "value": 129.4477982899989 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[13].total_s", "statistic": "single", "unit": "s", "value": 162.721182977024 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[13].decode_tps", "statistic": "single", "unit": "tok/s", "value": 15.357620055986176 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[13].user_number", "statistic": "single", "unit": "count", "value": 13 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[13].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40103 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[13].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[13].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[13].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[14].ttft_s", "statistic": "single", "unit": "s", "value": 92.11044140299782 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[14].total_s", "statistic": "single", "unit": "s", "value": 161.56033048901008 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[14].decode_tps", "statistic": "single", "unit": "tok/s", "value": 7.357823125781194 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[14].user_number", "statistic": "single", "unit": "count", "value": 14 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[14].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40105 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[14].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[14].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[14].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "team_load_requests[15].ttft_s", "statistic": "single", "unit": "s", "value": 61.292814782995265 }, { "basis": null, "n": null, "name": "total_s", "scope": "per_person", "source_field": "team_load_requests[15].total_s", "statistic": "single", "unit": "s", "value": 160.60114815901034 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "team_load_requests[15].decode_tps", "statistic": "single", "unit": "tok/s", "value": 5.1455903309260105 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "team_load_requests[15].user_number", "statistic": "single", "unit": "count", "value": 15 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "team_load_requests[15].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 40107 }, { "basis": null, "n": null, "name": "output_tokens", "scope": "per_person", "source_field": "team_load_requests[15].output_tokens", "statistic": "single", "unit": "tokens", "value": 512 }, { "basis": null, "n": null, "name": "needle_correct", "scope": "per_person", "source_field": "team_load_requests[15].needle_correct", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "needle_total", "scope": "per_person", "source_field": "team_load_requests[15].needle_total", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": 16, "name": "decode_tps", "scope": "per_person", "source_field": "per_person_decode_median_tps", "statistic": "median", "unit": "tok/s", "value": 7.509030611363343 } ], "model": { "artifact_sha256": [ "648fe83171485f4be03ea58d86089e088c94c2a3dc88922233f9002f30d8bc8b", "8303cef7d375021d69dcafc60d3fad5f8186995b18b0db5e2a609aadeb8a4f51", "cb5c5978721a1e87406ae68f1cd5a855326e40ff8bc13832644c392115336987" ], "publisher_repo": "ornith-ai/Ornith-1.5-35B-A3B", "publisher_revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1", "variant": "NVFP4 variant", "weights_dtype": "nvfp4", "weights_format": "safetensors" }, "outcome": { "gates": [ { "met": true, "name": "requests", "observed": "16/16" }, { "met": false, "name": "hidden-facts", "observed": "39/48" }, { "met": false, "name": "correctness-probes", "observed": "13/16" } ], "reasons": [ "39/48 Hidden facts found.", "13/16 probes passed." ], "status": "fail" }, "publication_group": "pre-standard", "published": null, "publisher": { "name": "Skilak Keep" }, "quality": [ { "passed": 39, "skipped": null, "suite": "hidden-facts", "total": 48, "version": "1" }, { "passed": 13, "skipped": null, "suite": "correctness-probes", "total": 16, "version": "1" } ], "record_date": "2026-10-07", "reproduce": { "driver": null, "fixture_sha256": "e2097959b1ca0142f2b93edc3ec411e474c0a6622800dd247af7a71bd9c39e83", "harness": { "name": "keep-team-load-v1", "version": "1" }, "harness_sha256": null, "os": null, "workload_name": null, "workload_version": null }, "requests": [ { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 11.972419038251711, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 162.35709048603894, "ttft_s": 119.6756575719919, "user_number": 0 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.223775013105565, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 160.66790649900213, "ttft_s": 62.84592934302054, "user_number": 1 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.223739365410741, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40103, "total_s": 160.66760278696893, "ttft_s": 62.84495807701023, "user_number": 2 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 10.994390057178135, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 162.2341767050093, "ttft_s": 115.75592758500716, "user_number": 3 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 13.824216319259065, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40107, "total_s": 162.55665049800882, "ttft_s": 125.59252984198974, "user_number": 4 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 9.198527422872003, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40102, "total_s": 161.97364485898288, "ttft_s": 106.4212747340207, "user_number": 5 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.22384230163479, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 160.66437300498364, "ttft_s": 62.8436558979447, "user_number": 6 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 7.508836126544265, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40101, "total_s": 161.61167190701235, "ttft_s": 93.55851541401353, "user_number": 7 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 11.751387300183673, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40106, "total_s": 162.33319285995094, "ttft_s": 118.84896525798831, "user_number": 8 }, { "cached_prompt_tokens": null, "checks_passed": false, "completed": true, "decode_tps": 15.059496363306511, "effective_prefill_tps": null, "needle_correct": 0, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40101, "total_s": 162.67787008499727, "ttft_s": 128.74579243297921, "user_number": 9 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 7.509225096182421, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40104, "total_s": 161.60899732500548, "ttft_s": 93.5593659110018, "user_number": 10 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.22393999810288, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40106, "total_s": 160.66355964302784, "ttft_s": 62.844671947997995, "user_number": 11 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.060898155385902, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40104, "total_s": 160.53207027696772, "ttft_s": 59.561850305995904, "user_number": 12 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 15.357620055986176, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40103, "total_s": 162.721182977024, "ttft_s": 129.4477982899989, "user_number": 13 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 7.357823125781194, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40105, "total_s": 161.56033048901008, "ttft_s": 92.11044140299782, "user_number": 14 }, { "cached_prompt_tokens": null, "checks_passed": true, "completed": true, "decode_tps": 5.1455903309260105, "effective_prefill_tps": null, "needle_correct": 3, "needle_total": 3, "output_tokens": 512, "prompt_tokens": 40107, "total_s": 160.60114815901034, "ttft_s": 61.292814782995265, "user_number": 15 } ], "scenario": { "id": "team-load", "version": "1" }, "settings": { "concurrency": 16, "configured_context_tokens": 262144, "context_tokens": 262144, "kv_dtype": "fp8", "max_output_tokens": 512, "speculative_decoding": "plain", "thinking": false }, "setup_complete": false, "setup_id": "a167e4bafd3fd96d0b47cf9d6645473d", "standard": "pre-standard", "system": { "driver": null, "hardware": "NVIDIA GB10, 128 GB unified memory", "os": null }, "workload": { "cache": null, "content_tokens": null, "document": null, "input_tokens": { "max": 40107, "median": 40104.5, "min": 40101 }, "output_tokens": 512, "repetitions": null, "users": 16, "warmup": null } }