{
  "caveats": [
    "Finite lab cohort; configured 262144 context differs from measured inputs. Aggregate uses first-output to final-completion window. Whole-host memory sampled during speed only; missing counters remain unassessed. Broader coding quality and delivery acceptance unassessed."
  ],
  "engine": {
    "name": "NVIDIA NGC vLLM",
    "recipe": "with multi-token prediction",
    "version": "0.29.0+5013de39.dev"
  },
  "format": "olir/0.1",
  "history": {
    "corrections": [],
    "supersedes": []
  },
  "id": "68118ad14a454699b5052087",
  "measured_at": "2026-10-07T06:41:57.846748+00:00",
  "metrics": [
    {
      "basis": null,
      "n": null,
      "name": "wall_seconds",
      "scope": "machine_total",
      "source_field": "wall_seconds",
      "statistic": "single",
      "unit": "s",
      "value": 159.96905493736267
    },
    {
      "basis": null,
      "n": 16,
      "name": "ttft_median_s",
      "scope": "per_person",
      "source_field": "ttft_median_s",
      "statistic": "median",
      "unit": "s",
      "value": 95.07330801652279
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "machine_total",
      "source_field": "needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 48
    },
    {
      "basis": null,
      "n": 16,
      "name": "ttft_max_s",
      "scope": "per_person",
      "source_field": "ttft_max_s",
      "statistic": "max",
      "unit": "s",
      "value": 135.08585100696655
    },
    {
      "basis": null,
      "n": null,
      "name": "requests_ok",
      "scope": "machine_total",
      "source_field": "requests_ok",
      "statistic": "single",
      "unit": "count",
      "value": 16
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "machine_total",
      "source_field": "needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 30
    },
    {
      "basis": null,
      "n": null,
      "name": "preemptions",
      "scope": "machine_total",
      "source_field": "preemptions",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_overlap",
      "scope": "machine_total",
      "source_field": "decode_overlap",
      "statistic": "single",
      "unit": "count",
      "value": 16
    },
    {
      "basis": null,
      "n": null,
      "name": "aggregate_concurrency",
      "scope": "machine_total",
      "source_field": "aggregate_concurrency",
      "statistic": "single",
      "unit": "count",
      "value": 16
    },
    {
      "basis": null,
      "n": null,
      "name": "requests_total",
      "scope": "machine_total",
      "source_field": "requests_total",
      "statistic": "single",
      "unit": "count",
      "value": 16
    },
    {
      "basis": null,
      "n": null,
      "name": "mtp_acceptance_rate",
      "scope": "machine_total",
      "source_field": "mtp_acceptance_rate",
      "statistic": "single",
      "unit": "ratio",
      "value": 0.7108307045215563
    },
    {
      "basis": null,
      "n": null,
      "name": "correctness_probes_passed",
      "scope": "machine_total",
      "source_field": "correctness_probes_passed",
      "statistic": "single",
      "unit": "count",
      "value": 10
    },
    {
      "basis": null,
      "n": null,
      "name": "aggregate_decode_tps",
      "scope": "machine_total",
      "source_field": "aggregate_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 80.96538619271466
    },
    {
      "basis": null,
      "n": null,
      "name": "aggregate_e2e_tps",
      "scope": "machine_total",
      "source_field": "aggregate_end_to_end_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 51.20990433560823
    },
    {
      "basis": null,
      "n": null,
      "name": "correctness_probes_total",
      "scope": "machine_total",
      "source_field": "correctness_probes_total",
      "statistic": "single",
      "unit": "count",
      "value": 16
    },
    {
      "basis": null,
      "n": 16,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "per_request_decode_tps",
      "statistic": "mean",
      "unit": "tok/s",
      "value": 10.431920443153471
    },
    {
      "basis": null,
      "n": null,
      "name": "forced_output_tokens",
      "scope": "machine_total",
      "source_field": "forced_output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "peak_host_unavailable_gib",
      "scope": "machine_total",
      "source_field": "peak_host_unavailable_gib",
      "statistic": "max",
      "unit": "GiB",
      "value": 93.93132400512695
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[0].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 135.08585100696655
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[0].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.96094153699232
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[0].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 20.542638805000184
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[0].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[0].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[0].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[0].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[0].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[1].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 95.0737348080147
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[1].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 158.83388462703442
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[1].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 8.014410277429558
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[1].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 1
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[1].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40105
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[1].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[1].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[1].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[2].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 95.07288122503087
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[2].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 157.9459811480483
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[2].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 8.127482192315544
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[2].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 2
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[2].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[2].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[2].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[2].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[3].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 62.64551535103237
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[3].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 156.7550567430444
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[3].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.429842632761713
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[3].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[3].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40102
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[3].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[3].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[3].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[4].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 134.18076786398888
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[4].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.61795602797065
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[4].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 20.088698353993358
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[4].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 4
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[4].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[4].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[4].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[4].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[5].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 58.98438179696677
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[5].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 156.21267304295907
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[5].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.2556719186511796
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[5].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 5
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[5].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[5].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[5].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[5].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[6].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 130.57057565398281
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[6].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.3935762999463
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[6].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 17.72889666404539
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[6].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 6
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[6].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40105
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[6].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[6].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[6].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[7].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 123.22267978900345
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[7].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.2675608129939
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[7].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 14.176770334181233
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[7].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 7
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[7].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[7].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[7].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[7].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[8].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 93.43093882902758
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[8].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 158.51897636504145
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[8].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 7.8509050102679545
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[8].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 8
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[8].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40099
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[8].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[8].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[8].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[9].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 123.22135551803512
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[9].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.396370794042
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[9].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 14.125771505587206
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[9].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 9
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[9].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[9].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[9].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[9].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[10].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 60.9555434640497
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[10].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 156.9136344169965
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[10].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.325241414510525
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[10].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 10
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[10].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40103
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[10].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[10].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[10].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[11].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 62.648104756954126
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[11].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 157.2439310689806
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[11].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.401929661404457
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[11].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 11
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[11].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40105
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[11].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[11].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[11].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[12].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 122.28557898901636
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[12].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.2178093570401
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[12].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 13.836153270678947
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[12].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 12
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[12].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40102
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[12].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[12].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[12].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[13].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 62.64641078998102
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[13].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 157.322281130997
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[13].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.397362582032921
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[13].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 13
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[13].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40101
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[13].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[13].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 0
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[13].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[14].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 62.64637023000978
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[14].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 157.53911308798706
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[14].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 5.385027185532999
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[14].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 14
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[14].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40100
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[14].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[14].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[14].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "ttft_s",
      "scope": "per_person",
      "source_field": "team_load_requests[15].ttft_s",
      "statistic": "single",
      "unit": "s",
      "value": 109.08666853199247
    },
    {
      "basis": null,
      "n": null,
      "name": "total_s",
      "scope": "per_person",
      "source_field": "team_load_requests[15].total_s",
      "statistic": "single",
      "unit": "s",
      "value": 159.06747198099038
    },
    {
      "basis": null,
      "n": null,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "team_load_requests[15].decode_tps",
      "statistic": "single",
      "unit": "tok/s",
      "value": 10.223925282062375
    },
    {
      "basis": null,
      "n": null,
      "name": "user_number",
      "scope": "per_person",
      "source_field": "team_load_requests[15].user_number",
      "statistic": "single",
      "unit": "count",
      "value": 15
    },
    {
      "basis": null,
      "n": null,
      "name": "prompt_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[15].prompt_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 40104
    },
    {
      "basis": null,
      "n": null,
      "name": "output_tokens",
      "scope": "per_person",
      "source_field": "team_load_requests[15].output_tokens",
      "statistic": "single",
      "unit": "tokens",
      "value": 512
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_correct",
      "scope": "per_person",
      "source_field": "team_load_requests[15].needle_correct",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": null,
      "name": "needle_total",
      "scope": "per_person",
      "source_field": "team_load_requests[15].needle_total",
      "statistic": "single",
      "unit": "count",
      "value": 3
    },
    {
      "basis": null,
      "n": 16,
      "name": "decode_tps",
      "scope": "per_person",
      "source_field": "per_person_decode_median_tps",
      "statistic": "median",
      "unit": "tok/s",
      "value": 8.070946234872551
    }
  ],
  "model": {
    "artifact_sha256": [
      "648fe83171485f4be03ea58d86089e088c94c2a3dc88922233f9002f30d8bc8b",
      "8303cef7d375021d69dcafc60d3fad5f8186995b18b0db5e2a609aadeb8a4f51",
      "cb5c5978721a1e87406ae68f1cd5a855326e40ff8bc13832644c392115336987"
    ],
    "publisher_repo": "ornith-ai/Ornith-1.5-35B-A3B",
    "publisher_revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1",
    "variant": "NVFP4 variant",
    "weights_dtype": "nvfp4",
    "weights_format": "safetensors"
  },
  "outcome": {
    "gates": [
      {
        "met": true,
        "name": "requests",
        "observed": "16/16"
      },
      {
        "met": false,
        "name": "hidden-facts",
        "observed": "30/48"
      },
      {
        "met": false,
        "name": "correctness-probes",
        "observed": "10/16"
      }
    ],
    "reasons": [
      "30/48 Hidden facts found.",
      "10/16 probes passed."
    ],
    "status": "fail"
  },
  "publication_group": "pre-standard",
  "published": null,
  "publisher": {
    "name": "Skilak Keep"
  },
  "quality": [
    {
      "passed": 30,
      "skipped": null,
      "suite": "hidden-facts",
      "total": 48,
      "version": "1"
    },
    {
      "passed": 10,
      "skipped": null,
      "suite": "correctness-probes",
      "total": 16,
      "version": "1"
    }
  ],
  "record_date": "2026-10-07",
  "reproduce": {
    "driver": null,
    "fixture_sha256": "e2097959b1ca0142f2b93edc3ec411e474c0a6622800dd247af7a71bd9c39e83",
    "harness": {
      "name": "keep-team-load-v1",
      "version": "1"
    },
    "harness_sha256": null,
    "os": null,
    "workload_name": null,
    "workload_version": null
  },
  "requests": [
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 20.542638805000184,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 159.96094153699232,
      "ttft_s": 135.08585100696655,
      "user_number": 0
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 8.014410277429558,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40105,
      "total_s": 158.83388462703442,
      "ttft_s": 95.0737348080147,
      "user_number": 1
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 8.127482192315544,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 157.9459811480483,
      "ttft_s": 95.07288122503087,
      "user_number": 2
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 5.429842632761713,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40102,
      "total_s": 156.7550567430444,
      "ttft_s": 62.64551535103237,
      "user_number": 3
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 20.088698353993358,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 159.61795602797065,
      "ttft_s": 134.18076786398888,
      "user_number": 4
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 5.2556719186511796,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 156.21267304295907,
      "ttft_s": 58.98438179696677,
      "user_number": 5
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 17.72889666404539,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40105,
      "total_s": 159.3935762999463,
      "ttft_s": 130.57057565398281,
      "user_number": 6
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 14.176770334181233,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 159.2675608129939,
      "ttft_s": 123.22267978900345,
      "user_number": 7
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 7.8509050102679545,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40099,
      "total_s": 158.51897636504145,
      "ttft_s": 93.43093882902758,
      "user_number": 8
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 14.125771505587206,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 159.396370794042,
      "ttft_s": 123.22135551803512,
      "user_number": 9
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 5.325241414510525,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40103,
      "total_s": 156.9136344169965,
      "ttft_s": 60.9555434640497,
      "user_number": 10
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 5.401929661404457,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40105,
      "total_s": 157.2439310689806,
      "ttft_s": 62.648104756954126,
      "user_number": 11
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 13.836153270678947,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40102,
      "total_s": 159.2178093570401,
      "ttft_s": 122.28557898901636,
      "user_number": 12
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": false,
      "completed": true,
      "decode_tps": 5.397362582032921,
      "effective_prefill_tps": null,
      "needle_correct": 0,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40101,
      "total_s": 157.322281130997,
      "ttft_s": 62.64641078998102,
      "user_number": 13
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 5.385027185532999,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40100,
      "total_s": 157.53911308798706,
      "ttft_s": 62.64637023000978,
      "user_number": 14
    },
    {
      "cached_prompt_tokens": null,
      "checks_passed": true,
      "completed": true,
      "decode_tps": 10.223925282062375,
      "effective_prefill_tps": null,
      "needle_correct": 3,
      "needle_total": 3,
      "output_tokens": 512,
      "prompt_tokens": 40104,
      "total_s": 159.06747198099038,
      "ttft_s": 109.08666853199247,
      "user_number": 15
    }
  ],
  "scenario": {
    "id": "team-load",
    "version": "1"
  },
  "settings": {
    "concurrency": 16,
    "configured_context_tokens": 262144,
    "context_tokens": 262144,
    "kv_dtype": "fp8",
    "max_output_tokens": 512,
    "speculative_decoding": "with multi-token prediction",
    "thinking": false
  },
  "setup_complete": false,
  "setup_id": "e682723b72283ed6b9af1479c6a4f677",
  "standard": "pre-standard",
  "system": {
    "driver": null,
    "hardware": "NVIDIA GB10, 128 GB unified memory",
    "os": null
  },
  "workload": {
    "cache": null,
    "content_tokens": null,
    "document": null,
    "input_tokens": {
      "max": 40105,
      "median": 40103,
      "min": 40099
    },
    "output_tokens": 512,
    "repetitions": null,
    "users": 16,
    "warmup": null
  }
}