{ "caveats": [ "Bounded tuning pilot; incomplete window growth cannot pass the full workload grade. Lab retrieval and rolling-history evidence only. Bounded literal user-source excerpts, with verified source hashes; details outside them remain unavailable. Shared engine cache intervals include overlapping users and are not per-user hits. Gateway path, not authenticated document ingestion or customer acceptance.", "Weights file SHA-256 was not recorded for public reproduction." ], "engine": { "name": "NVIDIA NGC vLLM", "recipe": "plain", "version": "0.29.0+5013de39.dev" }, "format": "olir/0.1", "history": { "corrections": [], "supersedes": [] }, "id": "e1c73db9297c42f0b08b2e20", "measured_at": "2026-10-07T17:10:05.103859+00:00", "metrics": [ { "basis": null, "n": null, "name": "roll_ttft_max_seconds", "scope": "per_person", "source_field": "roll_ttft_max_seconds", "statistic": "max", "unit": "s", "value": 4.2007552070426755 }, { "basis": null, "n": null, "name": "requests_ok", "scope": "machine_total", "source_field": "requests_ok", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[0].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[0].turn_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[0].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20195 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[0].ttft_s", "statistic": "single", "unit": "s", "value": 3.958434721978847 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[0].decode_tps", "statistic": "single", "unit": "tok/s", "value": 72.04116448387101 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[0].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[0].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[0].dropped_messages", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[0].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[1].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[1].turn_number", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[1].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20763 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[1].ttft_s", "statistic": "single", "unit": "s", "value": 4.162072690029163 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[1].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.54169700649555 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[1].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[1].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 40000 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[1].dropped_messages", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[1].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[2].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[2].turn_number", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[2].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20764 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[2].ttft_s", "statistic": "single", "unit": "s", "value": 4.1640529439901 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[2].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.82796060913925 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[2].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20001 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[2].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 60001 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[2].dropped_messages", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[2].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[3].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[3].turn_number", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[3].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20766 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[3].ttft_s", "statistic": "single", "unit": "s", "value": 4.157310820999555 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[3].decode_tps", "statistic": "single", "unit": "tok/s", "value": 72.05569044822327 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[3].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20003 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[3].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 80004 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[3].dropped_messages", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[3].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[4].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[4].turn_number", "statistic": "single", "unit": "count", "value": 5 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[4].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20762 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[4].ttft_s", "statistic": "single", "unit": "s", "value": 4.165260132984258 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[4].decode_tps", "statistic": "single", "unit": "tok/s", "value": 72.8985446793404 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[4].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[4].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 100004 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[4].dropped_messages", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[4].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[5].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[5].turn_number", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[5].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20765 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[5].ttft_s", "statistic": "single", "unit": "s", "value": 4.164349503989797 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[5].decode_tps", "statistic": "single", "unit": "tok/s", "value": 72.13545439166964 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[5].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20002 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[5].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 120006 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[5].dropped_messages", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[5].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[6].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[6].turn_number", "statistic": "single", "unit": "count", "value": 7 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[6].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20763 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[6].ttft_s", "statistic": "single", "unit": "s", "value": 4.164988242031541 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[6].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.91008875057591 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[6].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[6].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 140006 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[6].dropped_messages", "statistic": "single", "unit": "count", "value": 12 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[6].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[7].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[7].turn_number", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[7].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20762 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[7].ttft_s", "statistic": "single", "unit": "s", "value": 4.181012828019448 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[7].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.72233645720881 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[7].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[7].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 160006 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[7].dropped_messages", "statistic": "single", "unit": "count", "value": 14 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[7].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[8].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[8].turn_number", "statistic": "single", "unit": "count", "value": 9 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[8].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20763 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[8].ttft_s", "statistic": "single", "unit": "s", "value": 4.182111294998322 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[8].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.83691903034119 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[8].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[8].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 180006 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[8].dropped_messages", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[8].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[9].user_number", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[9].turn_number", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[9].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20767 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[9].ttft_s", "statistic": "single", "unit": "s", "value": 4.199595564976335 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[9].decode_tps", "statistic": "single", "unit": "tok/s", "value": 72.03129874076429 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[9].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20003 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[9].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 200009 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[9].dropped_messages", "statistic": "single", "unit": "count", "value": 18 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[9].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[10].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[10].turn_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[10].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20201 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[10].ttft_s", "statistic": "single", "unit": "s", "value": 3.9647572839749046 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[10].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.9626314688174 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[10].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20002 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[10].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 20002 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[10].dropped_messages", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[10].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[11].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[11].turn_number", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[11].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20753 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[11].ttft_s", "statistic": "single", "unit": "s", "value": 4.186333785997704 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[11].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.79376961586749 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[11].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[11].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 40002 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[11].dropped_messages", "statistic": "single", "unit": "count", "value": 2 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[11].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[12].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[12].turn_number", "statistic": "single", "unit": "count", "value": 3 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[12].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20753 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[12].ttft_s", "statistic": "single", "unit": "s", "value": 4.1741112260497175 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[12].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.62647244990436 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[12].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[12].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 60002 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[12].dropped_messages", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[12].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[13].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[13].turn_number", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[13].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20756 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[13].ttft_s", "statistic": "single", "unit": "s", "value": 4.17686991795199 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[13].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.6086736862249 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[13].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20002 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[13].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 80004 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[13].dropped_messages", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[13].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[14].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[14].turn_number", "statistic": "single", "unit": "count", "value": 5 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[14].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20757 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[14].ttft_s", "statistic": "single", "unit": "s", "value": 4.1770055330125615 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[14].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.31231452662743 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[14].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20003 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[14].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 100007 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[14].dropped_messages", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[14].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[15].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[15].turn_number", "statistic": "single", "unit": "count", "value": 6 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[15].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20753 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[15].ttft_s", "statistic": "single", "unit": "s", "value": 4.1750841560424305 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[15].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.50461339382697 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[15].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20000 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[15].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 120007 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[15].dropped_messages", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[15].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[16].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[16].turn_number", "statistic": "single", "unit": "count", "value": 7 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[16].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20757 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[16].ttft_s", "statistic": "single", "unit": "s", "value": 4.180532122962177 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[16].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.75668492738237 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[16].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20003 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[16].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 140010 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[16].dropped_messages", "statistic": "single", "unit": "count", "value": 12 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[16].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[17].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[17].turn_number", "statistic": "single", "unit": "count", "value": 8 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[17].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20755 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[17].ttft_s", "statistic": "single", "unit": "s", "value": 4.177881577983499 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[17].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.70952229442078 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[17].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20001 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[17].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 160011 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[17].dropped_messages", "statistic": "single", "unit": "count", "value": 14 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[17].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[18].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[18].turn_number", "statistic": "single", "unit": "count", "value": 9 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[18].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20755 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[18].ttft_s", "statistic": "single", "unit": "s", "value": 4.161982175020967 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[18].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.62464709020834 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[18].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20001 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[18].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 180012 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[18].dropped_messages", "statistic": "single", "unit": "count", "value": 16 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[18].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "user_number", "scope": "per_person", "source_field": "rolling_context_turns[19].user_number", "statistic": "single", "unit": "count", "value": 1 }, { "basis": null, "n": null, "name": "turn_number", "scope": "per_person", "source_field": "rolling_context_turns[19].turn_number", "statistic": "single", "unit": "count", "value": 10 }, { "basis": null, "n": null, "name": "prompt_tokens", "scope": "per_person", "source_field": "rolling_context_turns[19].prompt_tokens", "statistic": "single", "unit": "tokens", "value": 20757 }, { "basis": null, "n": null, "name": "ttft_s", "scope": "per_person", "source_field": "rolling_context_turns[19].ttft_s", "statistic": "single", "unit": "s", "value": 4.2007552070426755 }, { "basis": null, "n": null, "name": "decode_tps", "scope": "per_person", "source_field": "rolling_context_turns[19].decode_tps", "statistic": "single", "unit": "tok/s", "value": 71.99682203356258 }, { "basis": null, "n": null, "name": "content_tokens_added", "scope": "per_person", "source_field": "rolling_context_turns[19].content_tokens_added", "statistic": "single", "unit": "tokens", "value": 20002 }, { "basis": null, "n": null, "name": "cumulative_content_tokens", "scope": "per_person", "source_field": "rolling_context_turns[19].cumulative_content_tokens", "statistic": "single", "unit": "tokens", "value": 200014 }, { "basis": null, "n": null, "name": "dropped_messages", "scope": "per_person", "source_field": "rolling_context_turns[19].dropped_messages", "statistic": "single", "unit": "count", "value": 18 }, { "basis": null, "n": null, "name": "engine_interval_cache_hit_rate", "scope": "machine_total", "source_field": "rolling_context_turns[19].engine_interval_cache_hit_rate", "statistic": "single", "unit": "ratio", "value": 0 }, { "basis": null, "n": null, "name": "roll_ttft_p50_seconds", "scope": "per_person", "source_field": "roll_ttft_p50_seconds", "statistic": "median", "unit": "s", "value": 4.17597703699721 }, { "basis": null, "n": null, "name": "rolled_off_abstained", "scope": "machine_total", "source_field": "rolled_off_abstained", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "rolled_off_invalid", "scope": "machine_total", "source_field": "rolled_off_invalid", "statistic": "single", "unit": "count", "value": 0 }, { "basis": null, "n": null, "name": "rolled_off_wrong", "scope": "machine_total", "source_field": "rolled_off_wrong", "statistic": "single", "unit": "count", "value": 4 }, { "basis": null, "n": null, "name": "normal_ttft_max_seconds", "scope": "per_person", "source_field": "normal_ttft_max_seconds", "statistic": "max", "unit": "s", "value": 3.9647572839749046 }, { "basis": null, "n": null, "name": "rolled_off_correct", "scope": "machine_total", "source_field": "rolled_off_correct", "statistic": "single", "unit": "count", "value": 14 }, { "basis": null, "n": null, "name": "normal_ttft_p50_seconds", "scope": "per_person", "source_field": "normal_ttft_p50_seconds", "statistic": "median", "unit": "s", "value": 3.9615960029768758 }, { "basis": null, "n": null, "name": "rolled_off_total", "scope": "machine_total", "source_field": "rolled_off_total", "statistic": "single", "unit": "count", "value": 18 }, { "basis": null, "n": null, "name": "requests_total", "scope": "machine_total", "source_field": "requests_total", "statistic": "single", "unit": "count", "value": 20 } ], "model": { "artifact_sha256": [], "publisher_repo": "Qwen/Qwen3.6-35B-A3B", "publisher_revision": "995ad96eacd98c81ed38be0c5b274b04031597b0", "variant": "NVFP4 variant", "weights_dtype": "NVFP4", "weights_format": "safetensors" }, "outcome": { "gates": [ { "met": false, "name": "requests", "observed": "16/20" } ], "reasons": [ "16/20 requests passed." ], "status": "fail" }, "publication_group": "pre-standard", "published": null, "publisher": { "name": "Skilak Keep" }, "quality": [], "record_date": "2026-10-07", "reproduce": { "driver": null, "fixture_sha256": null, "harness": { "name": "keep-rolling-context-v1", "version": "1" }, "harness_sha256": null, "os": null, "workload_name": null, "workload_version": null }, "requests": [], "scenario": { "id": "rolling-context", "version": "1" }, "settings": { "concurrency": 2, "configured_context_tokens": 262144, "context_tokens": 262144, "max_output_tokens": 256 }, "setup_complete": false, "setup_id": "47a03fd9bbfe612c42f992e302c0e2fe", "standard": "pre-standard", "system": { "driver": null, "hardware": "NVIDIA GB10, 128 GB unified memory", "os": null }, "workload": { "cache": null, "content_tokens": null, "document": null, "input_tokens": null, "output_tokens": 256, "repetitions": null, "users": 2, "warmup": null } }