{ "article_key": "llm-context-concurrency", "cases": [ { "case_wall_seconds": 31.54574643599335, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 5752.575337435909, "reason": null, "response_union_seconds": 19.211848054022994, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.428525629595, "trapezoid_pieces": 29, "watt_hours": 1.597937593732197 }, "gpu_sample_count": 19, "host_sample_count": 19, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16754032640, "mean": 16745837837.473684, "median": 16747278336, "minimum": 16736788480, "sample_count": 19 }, "gpu_board_watts": { "maximum": 300.75, "mean": 299.9973684210526, "median": 299.98, "minimum": 299.21, "sample_count": 19 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 19 }, "gpu_sm_clock_mhz": { "maximum": 1845.0, "mean": 1788.3157894736842, "median": 1777.0, "minimum": 1695.0, "sample_count": 19 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 97.73684210526316, "median": 97.0, "minimum": 97.0, "sample_count": 19 }, "host_mem_available_bytes": { "maximum": 96162541568, "mean": 96125036220.63158, "median": 96112386048, "minimum": 96093450240, "sample_count": 19 }, "process_pss_bytes": { "maximum": 821585920, "mean": 817439474.5263158, "median": 818100224, "minimum": 812947456, "sample_count": 19 }, "process_rss_bytes": { "maximum": 824573952, "mean": 820427506.5263158, "median": 821088256, "minimum": 815935488, "sample_count": 19 } }, "process_sample_count": 19, "response_union_seconds": 19.211848054022994, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 9.185685446020216, "observed_intervals": 7, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 171.87876975330158, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.120186341505504, "interval_count": 5, "interval_seconds": 7.461493466049433, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.84596292817612, "process_cpu_seconds": 7.45 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 2226.7114520086775, "reason": null, "response_union_seconds": 7.447109389002435, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.00345700534325, "trapezoid_pieces": 12, "watt_hours": 0.6185309588912993 }, "gpu_sample_count": 7, "host_sample_count": 7, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16740397056, "mean": 16738646308.571428, "median": 16738406400, "minimum": 16736788480, "sample_count": 7 }, "gpu_board_watts": { "maximum": 300.11, "mean": 299.9857142857143, "median": 299.97, "minimum": 299.91, "sample_count": 7 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 7 }, "gpu_sm_clock_mhz": { "maximum": 1845.0, "mean": 1833.0, "median": 1830.0, "minimum": 1822.0, "sample_count": 7 }, "gpu_utilization_percent": { "maximum": 97.0, "mean": 97.0, "median": 97.0, "minimum": 97.0, "sample_count": 7 }, "host_mem_available_bytes": { "maximum": 96162541568, "mean": 96152399286.85715, "median": 96153411584, "minimum": 96136941568, "sample_count": 7 }, "process_pss_bytes": { "maximum": 814241792, "mean": 813606326.8571428, "median": 813594624, "minimum": 812947456, "sample_count": 7 }, "process_rss_bytes": { "maximum": 817229824, "mean": 816594358.8571428, "median": 816582656, "minimum": 815935488, "sample_count": 7 } }, "process_sample_count": 7, "response_union_seconds": 7.447109389002435, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 2.034676303039305, "observed_intervals": 2, "requested_intervals": 5 } }, "response_outer_span_seconds": 7.469806003966369, "response_seconds": { "maximum": 1.4900457450421527, "mean": 1.489421877800487, "median": 1.4897573189809918, "minimum": 1.4879181029973552, "sample_count": 5 }, "response_window_union_seconds": 7.447109389002435, "server_decode_tokens_per_second": { "maximum": 196.57875870150093, "mean": 196.4327494545978, "median": 196.40161434424968, "minimum": 196.36970177217881, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 0.19175322900991887, "mean": 0.19115480119362474, "median": 0.1911611080286093, "minimum": 0.1905973819666542, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 }, { "aggregate_output_tokens_per_second": 108.79969682673467, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.118724668523497, "interval_count": 5, "interval_seconds": 11.783662844914943, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.7991893927519, "process_cpu_seconds": 11.76 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 3525.863885427233, "reason": null, "response_union_seconds": 11.764738665020559, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.6975951459498, "trapezoid_pieces": 17, "watt_hours": 0.9794066348408981 }, "gpu_sample_count": 12, "host_sample_count": 12, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16754032640, "mean": 16750032896, "median": 16750100480.0, "minimum": 16746192896, "sample_count": 12 }, "gpu_board_watts": { "maximum": 300.75, "mean": 300.00416666666666, "median": 300.04499999999996, "minimum": 299.21, "sample_count": 12 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 12 }, "gpu_sm_clock_mhz": { "maximum": 1822.0, "mean": 1762.25, "median": 1766.0, "minimum": 1695.0, "sample_count": 12 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 98.16666666666667, "median": 97.5, "minimum": 97.0, "sample_count": 12 }, "host_mem_available_bytes": { "maximum": 96129368064, "mean": 96109074432, "median": 96109725696.0, "minimum": 96093450240, "sample_count": 12 }, "process_pss_bytes": { "maximum": 821585920, "mean": 819675477.3333334, "median": 819517440.0, "minimum": 818100224, "sample_count": 12 }, "process_rss_bytes": { "maximum": 824573952, "mean": 822663509.3333334, "median": 822505472.0, "minimum": 821088256, "sample_count": 12 } }, "process_sample_count": 12, "response_union_seconds": 11.764738665020559, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 7.151009142980911, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 11.794427076005377, "response_seconds": { "maximum": 2.3576946139801294, "mean": 2.352947733004112, "median": 2.354433296015486, "minimum": 2.34604896401288, "sample_count": 5 }, "response_window_union_seconds": 11.764738665020559, "server_decode_tokens_per_second": { "maximum": 168.74107905618814, "mean": 168.25912788557528, "median": 168.20414046035162, "minimum": 167.88277437147653, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 0.8393656879779883, "mean": 0.8372448727837764, "median": 0.8369915079674684, "minimum": 0.8346778709674254, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 } ], "id": "concurrency-8b-c1", "model": "qwen3-8b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 333.84 }, { "device": "CUDA0", "reported_mib": 4455.34 } ], "gpu_layers": 37, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 37 }, "record_sha256": "cfb0005252065662c3806270e4fb851d33ec2bc1e4fcec950a2872a8f015cbd6", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 15503560704, "mean": 15494721536, "median": 15494721536.0, "minimum": 15485882368, "sample_count": 2 }, "gpu_board_watts": { "maximum": 15.74, "mean": 15.645, "median": 15.645, "minimum": 15.55, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96575950848, "mean": 96570210304, "median": 96570210304.0, "minimum": 96564469760, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1004508669720963, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16422359040, "mean": 16422359040, "median": 16422359040, "minimum": 16422359040, "sample_count": 1 }, "gpu_board_watts": { "maximum": 81.37, "mean": 81.37, "median": 81.37, "minimum": 81.37, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2332.0, "mean": 2332.0, "median": 2332.0, "minimum": 2332.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96489291776, "mean": 96489291776, "median": 96489291776, "minimum": 96489291776, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1638742850045674, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 140.7218834464672, "reason": null, "response_union_seconds": 2.7998547020251863, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 50.26042363722677, "trapezoid_pieces": 3, "watt_hours": 0.03908941206846311 }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16758321152, "mean": 16441323520, "median": 16441323520.0, "minimum": 16124325888, "sample_count": 2 }, "gpu_board_watts": { "maximum": 68.68, "mean": 52.83500000000001, "median": 52.83500000000001, "minimum": 36.99, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 5259657216.0, "mean": 2923429888.0, "median": 2923429888.0, "minimum": 587202560.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 2272.0, "mean": 2272.0, "median": 2272.0, "minimum": 2272.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96308260864, "mean": 96301864960, "median": 96301864960.0, "minimum": 96295469056, "sample_count": 2 }, "process_pss_bytes": { "maximum": 5363132416, "mean": 3241467904, "median": 3241467904.0, "minimum": 1119803392, "sample_count": 2 }, "process_rss_bytes": { "maximum": 5366120448, "mean": 3244455936, "median": 3244455936.0, "minimum": 1122791424, "sample_count": 2 } }, "process_sample_count": 2, "response_union_seconds": 2.7998547020251863, "sampled_counter_deltas": { "deltas": { "major_faults": 1, "read_syscall_bytes": 5, "storage_read_bytes": 988917760, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 1.070253963000141, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 485.16816934820196, "reason": null, "response_union_seconds": 3.326905186928343, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 145.83167901942733, "trapezoid_pieces": 8, "watt_hours": 0.1347689359300561 }, "gpu_sample_count": 5, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16755085312, "mean": 16734448298.666666, "median": 16740896768, "minimum": 16707362816, "sample_count": 3 }, "gpu_board_watts": { "maximum": 300.08, "mean": 133.35399999999998, "median": 78.6, "minimum": 75.12, "sample_count": 5 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10595650764.8, "median": 10603200512.0, "minimum": 10584326144.0, "sample_count": 5 }, "gpu_sm_clock_mhz": { "maximum": 2332.0, "mean": 2213.6, "median": 2302.0, "minimum": 1822.0, "sample_count": 5 }, "gpu_utilization_percent": { "maximum": 97.0, "mean": 19.4, "median": 0.0, "minimum": 0.0, "sample_count": 5 }, "host_mem_available_bytes": { "maximum": 96211808256, "mean": 96155060906.66667, "median": 96133586944, "minimum": 96119787520, "sample_count": 3 }, "process_pss_bytes": { "maximum": 821585920, "mean": 805507754.6666666, "median": 814356480, "minimum": 780580864, "sample_count": 3 }, "process_rss_bytes": { "maximum": 824573952, "mean": 808495786.6666666, "median": 817344512, "minimum": 783568896, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.326905186928343, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 5771.959698463556, "reason": null, "response_union_seconds": 19.277330336975865, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.4169627000869, "trapezoid_pieces": 21, "watt_hours": 1.603322138462099 }, "gpu_sample_count": 19, "host_sample_count": 19, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16754032640, "mean": 16745837837.473684, "median": 16747278336, "minimum": 16736788480, "sample_count": 19 }, "gpu_board_watts": { "maximum": 300.75, "mean": 299.9973684210526, "median": 299.98, "minimum": 299.21, "sample_count": 19 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 19 }, "gpu_sm_clock_mhz": { "maximum": 1845.0, "mean": 1788.3157894736842, "median": 1777.0, "minimum": 1695.0, "sample_count": 19 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 97.73684210526316, "median": 97.0, "minimum": 97.0, "sample_count": 19 }, "host_mem_available_bytes": { "maximum": 96162541568, "mean": 96125036220.63158, "median": 96112386048, "minimum": 96093450240, "sample_count": 19 }, "process_pss_bytes": { "maximum": 821585920, "mean": 817439474.5263158, "median": 818100224, "minimum": 812947456, "sample_count": 19 }, "process_rss_bytes": { "maximum": 824573952, "mean": 820427506.5263158, "median": 821088256, "minimum": 815935488, "sample_count": 19 } }, "process_sample_count": 19, "response_union_seconds": 19.277330336975865, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 16384 }, "observed_endpoint_seconds": 17.382738566957414, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 872.0933225952482, "reason": null, "response_union_seconds": 3.8364628920680843, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 227.31702277071614, "trapezoid_pieces": 5, "watt_hours": 0.24224814516534673 }, "gpu_sample_count": 3, "host_sample_count": 4, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16744005632, "mean": 16737950720, "median": 16738725888.0, "minimum": 16730345472, "sample_count": 4 }, "gpu_board_watts": { "maximum": 300.32, "mean": 273.2, "median": 277.67, "minimum": 241.61, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 1837.0, "mean": 1782.0, "median": 1777.0, "minimum": 1732.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 98.0, "mean": 97.33333333333333, "median": 97.0, "minimum": 97.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96168611840, "mean": 96138284032, "median": 96137695232.0, "minimum": 96109133824, "sample_count": 4 }, "process_pss_bytes": { "maximum": 817391616, "mean": 812983296, "median": 814118912.0, "minimum": 806303744, "sample_count": 4 }, "process_rss_bytes": { "maximum": 820379648, "mean": 815971328, "median": 817106944.0, "minimum": 809291776, "sample_count": 4 } }, "process_sample_count": 4, "response_union_seconds": 3.8364628920680843, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 2.0357347379322164, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 1, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 32, "host_samples": 31, "maximum_gpu_gap_seconds": 1.0082540499861352, "maximum_host_gap_seconds": 1.070253963000141, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 32, "unavailable": 0 }, "software": { "active": 0, "not_active": 32, "unavailable": 0 } } } }, { "case_wall_seconds": 143.20899132004706, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 14770.156143266022, "reason": null, "response_union_seconds": 49.281771619047504, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.70830305047974, "trapezoid_pieces": 60, "watt_hours": 4.1028211509072285 }, "gpu_sample_count": 50, "host_sample_count": 48, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16972275712, "mean": 16944400810.666666, "median": 16953174016.0, "minimum": 16903892992, "sample_count": 48 }, "gpu_board_watts": { "maximum": 300.93, "mean": 299.8862, "median": 299.99, "minimum": 296.32, "sample_count": 50 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 10604584632.32, "median": 10605297664.0, "minimum": 10603200512.0, "sample_count": 50 }, "gpu_sm_clock_mhz": { "maximum": 1860.0, "mean": 1748.2, "median": 1751.0, "minimum": 1680.0, "sample_count": 50 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 96.26, "median": 96.0, "minimum": 77.0, "sample_count": 50 }, "host_mem_available_bytes": { "maximum": 96069586944, "mean": 96035869354.66667, "median": 96034033664.0, "minimum": 96010240000, "sample_count": 48 }, "process_pss_bytes": { "maximum": 858840064, "mean": 846069888, "median": 851729408.0, "minimum": 827268096, "sample_count": 48 }, "process_rss_bytes": { "maximum": 861831168, "mean": 849090560, "median": 854720512.0, "minimum": 830259200, "sample_count": 48 } }, "process_sample_count": 48, "response_union_seconds": 49.281771619047504, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 28672 }, "observed_endpoint_seconds": 38.876957018102985, "observed_intervals": 10, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 313.01834665113336, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1419778907509697, "interval_count": 5, "interval_seconds": 16.371057273005135, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 100.54329250403103, "process_cpu_seconds": 16.46 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 4901.074335239871, "reason": null, "response_union_seconds": 16.356868710019626, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.6340205529467, "trapezoid_pieces": 22, "watt_hours": 1.3614095375666309 }, "gpu_sample_count": 17, "host_sample_count": 16, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16922185728, "mean": 16913083904, "median": 16913446912.0, "minimum": 16903892992, "sample_count": 16 }, "gpu_board_watts": { "maximum": 300.33, "mean": 299.7952941176471, "median": 300.02, "minimum": 296.32, "sample_count": 17 }, "gpu_memory_used_bytes": { "maximum": 10603200512.0, "mean": 10603200512.0, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 17 }, "gpu_sm_clock_mhz": { "maximum": 1860.0, "mean": 1759.2941176470588, "median": 1755.0, "minimum": 1725.0, "sample_count": 17 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 95.05882352941177, "median": 96.0, "minimum": 77.0, "sample_count": 17 }, "host_mem_available_bytes": { "maximum": 96069586944, "mean": 96051451904, "median": 96053071872.0, "minimum": 96031924224, "sample_count": 16 }, "process_pss_bytes": { "maximum": 832879616, "mean": 830089728, "median": 830098432.0, "minimum": 827268096, "sample_count": 16 }, "process_rss_bytes": { "maximum": 835870720, "mean": 833080832, "median": 833089536.0, "minimum": 830259200, "sample_count": 16 } }, "process_sample_count": 16, "response_union_seconds": 16.356868710019626, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 4096 }, "observed_endpoint_seconds": 11.232935156032909, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 16.403912629000843, "response_seconds": { "maximum": 3.28185669897357, "mean": 3.258913619956002, "median": 3.2634908619802445, "minimum": 3.2179629480233416, "sample_count": 20 }, "response_window_union_seconds": 16.356868710019626, "server_decode_tokens_per_second": { "maximum": 104.92626769447776, "mean": 96.54218459985283, "median": 99.48448007706719, "minimum": 83.302244717821, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 0.825031254964415, "mean": 0.5970895664999262, "median": 0.692155514494516, "minimum": 0.18841045402223244, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 }, { "aggregate_output_tokens_per_second": 155.5053940218641, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1331217145339085, "interval_count": 5, "interval_seconds": 32.94437924999511, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 100.25989486508507, "process_cpu_seconds": 33.03 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 9869.081808026149, "reason": null, "response_union_seconds": 32.92490290902788, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.7452060920151, "trapezoid_pieces": 38, "watt_hours": 2.741411613340597 }, "gpu_sample_count": 33, "host_sample_count": 32, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16972275712, "mean": 16960059264, "median": 16959395840.0, "minimum": 16945074176, "sample_count": 32 }, "gpu_board_watts": { "maximum": 300.93, "mean": 299.9330303030303, "median": 299.97, "minimum": 298.18, "sample_count": 33 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 10605297664.0, "median": 10605297664.0, "minimum": 10605297664.0, "sample_count": 33 }, "gpu_sm_clock_mhz": { "maximum": 1845.0, "mean": 1742.4848484848485, "median": 1725.0, "minimum": 1680.0, "sample_count": 33 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 96.87878787878788, "median": 96.0, "minimum": 79.0, "sample_count": 33 }, "host_mem_available_bytes": { "maximum": 96050204672, "mean": 96028078080, "median": 96025366528.0, "minimum": 96010240000, "sample_count": 32 }, "process_pss_bytes": { "maximum": 858840064, "mean": 854059968, "median": 854100992.0, "minimum": 847817728, "sample_count": 32 }, "process_rss_bytes": { "maximum": 861831168, "mean": 857095424, "median": 857092096.0, "minimum": 850808832, "sample_count": 32 } }, "process_sample_count": 32, "response_union_seconds": 32.92490290902788, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 24576 }, "observed_endpoint_seconds": 27.644021862070076, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 32.988753579964396, "response_seconds": { "maximum": 6.621589451970067, "mean": 6.539755831405637, "median": 6.544440800498705, "minimum": 6.456752622034401, "sample_count": 20 }, "response_window_union_seconds": 32.92490290902788, "server_decode_tokens_per_second": { "maximum": 84.3563330599747, "mean": 64.69884639525425, "median": 63.226577706461605, "minimum": 46.57408347683021, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 3.577183506975416, "mean": 2.4010472706053405, "median": 2.47954356399714, "minimum": 1.0299478119704872, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 } ], "id": "concurrency-8b-c4", "model": "qwen3-8b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 333.84 }, { "device": "CUDA0", "reported_mib": 4455.34 } ], "gpu_layers": 37, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 37 }, "record_sha256": "546c5e177b7e76804c6d4a2aedb3859980726dc415bfddd3c1a7373c9774519f", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16447270912, "mean": 16438515712, "median": 16438515712.0, "minimum": 16429760512, "sample_count": 2 }, "gpu_board_watts": { "maximum": 16.25, "mean": 16.244999999999997, "median": 16.244999999999997, "minimum": 16.24, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96482447360, "mean": 96481126400, "median": 96481126400.0, "minimum": 96479805440, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1007217759615742, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16606367744, "mean": 16606367744, "median": 16606367744, "minimum": 16606367744, "sample_count": 1 }, "gpu_board_watts": { "maximum": 90.64, "mean": 90.15, "median": 90.15, "minimum": 89.66, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 5303697408.0, "median": 5303697408.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2328.5, "median": 2328.5, "minimum": 2317.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96498147328, "mean": 96498147328, "median": 96498147328, "minimum": 96498147328, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1552442620159127, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 53.5790935900832, "reason": null, "response_union_seconds": 1.3030608020490035, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 41.11787685258625, "trapezoid_pieces": 2, "watt_hours": 0.014883081552800888 }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16777781248, "mean": 16777781248, "median": 16777781248, "minimum": 16777781248, "sample_count": 1 }, "gpu_board_watts": { "maximum": 50.83, "mean": 50.83, "median": 50.83, "minimum": 50.83, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 5259657216.0, "mean": 5259657216.0, "median": 5259657216.0, "minimum": 5259657216.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2287.0, "mean": 2287.0, "median": 2287.0, "minimum": 2287.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 76.0, "mean": 76.0, "median": 76.0, "minimum": 76.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96183443456, "mean": 96183443456, "median": 96183443456, "minimum": 96183443456, "sample_count": 1 }, "process_pss_bytes": { "maximum": 5363969024, "mean": 5363969024, "median": 5363969024, "minimum": 5363969024, "sample_count": 1 }, "process_rss_bytes": { "maximum": 5366960128, "mean": 5366960128, "median": 5366960128, "minimum": 5366960128, "sample_count": 1 } }, "process_sample_count": 1, "response_union_seconds": 1.3030608020490035, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 528.2204930815988, "reason": null, "response_union_seconds": 3.318441729003098, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 159.1772693987557, "trapezoid_pieces": 6, "watt_hours": 0.14672791474488855 }, "gpu_sample_count": 3, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16974327808, "mean": 16917742933.333334, "median": 16924258304, "minimum": 16854642688, "sample_count": 3 }, "gpu_board_watts": { "maximum": 280.61, "mean": 165.19, "median": 134.2, "minimum": 80.76, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 10597608106.666666, "median": 10603200512.0, "minimum": 10584326144.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2332.0, "mean": 2149.6666666666665, "median": 2310.0, "minimum": 1807.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 1.0, "mean": 0.3333333333333333, "median": 0.0, "minimum": 0.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96106070016, "mean": 96071060138.66667, "median": 96063447040, "minimum": 96043663360, "sample_count": 3 }, "process_pss_bytes": { "maximum": 858840064, "mean": 824471893.3333334, "median": 833178624, "minimum": 781396992, "sample_count": 3 }, "process_rss_bytes": { "maximum": 861831168, "mean": 827462997.3333334, "median": 836169728, "minimum": 784388096, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.318441729003098, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 14811.261750294023, "reason": null, "response_union_seconds": 49.420940532989334, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.6960719597647, "trapezoid_pieces": 52, "watt_hours": 4.114239375081673 }, "gpu_sample_count": 50, "host_sample_count": 48, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16972275712, "mean": 16944400810.666666, "median": 16953174016.0, "minimum": 16903892992, "sample_count": 48 }, "gpu_board_watts": { "maximum": 300.93, "mean": 299.8862, "median": 299.99, "minimum": 296.32, "sample_count": 50 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 10604584632.32, "median": 10605297664.0, "minimum": 10603200512.0, "sample_count": 50 }, "gpu_sm_clock_mhz": { "maximum": 1860.0, "mean": 1748.2, "median": 1751.0, "minimum": 1680.0, "sample_count": 50 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 96.26, "median": 96.0, "minimum": 77.0, "sample_count": 50 }, "host_mem_available_bytes": { "maximum": 96069586944, "mean": 96035869354.66667, "median": 96034033664.0, "minimum": 96010240000, "sample_count": 48 }, "process_pss_bytes": { "maximum": 858840064, "mean": 846069888, "median": 851729408.0, "minimum": 827268096, "sample_count": 48 }, "process_rss_bytes": { "maximum": 861831168, "mean": 849090560, "median": 854720512.0, "minimum": 830259200, "sample_count": 48 } }, "process_sample_count": 48, "response_union_seconds": 49.420940532989334, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 86016 }, "observed_endpoint_seconds": 47.11209448100999, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 2686.3683206647843, "reason": null, "response_union_seconds": 9.834585551987402, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 273.15521396038037, "trapezoid_pieces": 11, "watt_hours": 0.7462134224068845 }, "gpu_sample_count": 9, "host_sample_count": 10, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16940253184, "mean": 16923122892.8, "median": 16929480704.0, "minimum": 16887836672, "sample_count": 10 }, "gpu_board_watts": { "maximum": 300.17, "mean": 276.2988888888889, "median": 299.98, "minimum": 174.1, "sample_count": 9 }, "gpu_memory_used_bytes": { "maximum": 10605297664.0, "mean": 10604132579.555555, "median": 10603200512.0, "minimum": 10603200512.0, "sample_count": 9 }, "gpu_sm_clock_mhz": { "maximum": 1800.0, "mean": 1750.5555555555557, "median": 1747.0, "minimum": 1717.0, "sample_count": 9 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 96.22222222222223, "median": 96.0, "minimum": 92.0, "sample_count": 9 }, "host_mem_available_bytes": { "maximum": 96077516800, "mean": 96048424140.8, "median": 96048119808.0, "minimum": 96022204416, "sample_count": 10 }, "process_pss_bytes": { "maximum": 845200384, "mean": 835459686.4, "median": 835929088.0, "minimum": 816208896, "sample_count": 10 }, "process_rss_bytes": { "maximum": 848191488, "mean": 838450790.4, "median": 838920192.0, "minimum": 819200000, "sample_count": 10 } }, "process_sample_count": 10, "response_union_seconds": 9.834585551987402, "sampled_counter_deltas": { "deltas": { "major_faults": 16, "read_syscall_bytes": 0, "storage_read_bytes": 2822144, "storage_write_bytes": 8192 }, "observed_endpoint_seconds": 8.234222369035706, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 4, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 67, "host_samples": 65, "maximum_gpu_gap_seconds": 1.059977461991366, "maximum_host_gap_seconds": 1.092143846966792, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 67, "unavailable": 0 }, "software": { "active": 0, "not_active": 67, "unavailable": 0 } } } }, { "case_wall_seconds": 156.2884025030071, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 20792.72662546744, "reason": null, "response_union_seconds": 69.32143143203575, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9465850016828, "trapezoid_pieces": 78, "watt_hours": 5.775757395963177 }, "gpu_sample_count": 68, "host_sample_count": 68, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17265410048, "mean": 17253726268.235294, "median": 17256364032.0, "minimum": 17243848704, "sample_count": 68 }, "gpu_board_watts": { "maximum": 301.23, "mean": 299.98058823529414, "median": 299.99, "minimum": 297.88, "sample_count": 68 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29347545088.0, "median": 29347545088.0, "minimum": 29347545088.0, "sample_count": 68 }, "gpu_sm_clock_mhz": { "maximum": 1717.0, "mean": 1611.6176470588234, "median": 1605.0, "minimum": 1575.0, "sample_count": 68 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.08823529411765, "median": 99.0, "minimum": 84.0, "sample_count": 68 }, "host_mem_available_bytes": { "maximum": 96128315392, "mean": 96101368591.05882, "median": 96100413440.0, "minimum": 96063385600, "sample_count": 68 }, "process_pss_bytes": { "maximum": 932377600, "mean": 922582874.3529412, "median": 924959744.0, "minimum": 913896448, "sample_count": 68 }, "process_rss_bytes": { "maximum": 935366656, "mean": 925592696.4705882, "median": 927948800.0, "minimum": 917929984, "sample_count": 68 } }, "process_sample_count": 68, "response_union_seconds": 69.32143143203575, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 40960 }, "observed_endpoint_seconds": 59.37992162600858, "observed_intervals": 10, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 46.203362423998605, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1206085574221407, "interval_count": 5, "interval_seconds": 27.718952380062547, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.8594738375085, "process_cpu_seconds": 27.68 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 8309.739476456769, "reason": null, "response_union_seconds": 27.70361144398339, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9514880310473, "trapezoid_pieces": 32, "watt_hours": 2.308260965682436 }, "gpu_sample_count": 27, "host_sample_count": 27, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17247604736, "mean": 17245643510.51852, "median": 17245782016, "minimum": 17243848704, "sample_count": 27 }, "gpu_board_watts": { "maximum": 301.23, "mean": 300.0292592592593, "median": 299.99, "minimum": 299.65, "sample_count": 27 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29347545088.0, "median": 29347545088.0, "minimum": 29347545088.0, "sample_count": 27 }, "gpu_sm_clock_mhz": { "maximum": 1717.0, "mean": 1612.5925925925926, "median": 1605.0, "minimum": 1575.0, "sample_count": 27 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.18518518518519, "median": 99.0, "minimum": 99.0, "sample_count": 27 }, "host_mem_available_bytes": { "maximum": 96128315392, "mean": 96108294902.51852, "median": 96110452736, "minimum": 96087101440, "sample_count": 27 }, "process_pss_bytes": { "maximum": 916280320, "mean": 915551725.037037, "median": 915596288, "minimum": 913896448, "sample_count": 27 }, "process_rss_bytes": { "maximum": 919269376, "mean": 918593080.8888888, "median": 918585344, "minimum": 917929984, "sample_count": 27 } }, "process_sample_count": 27, "response_union_seconds": 27.70361144398339, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 20480 }, "observed_endpoint_seconds": 22.517533548991196, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 27.72946648998186, "response_seconds": { "maximum": 5.577194019977469, "mean": 5.540722288796678, "median": 5.540344331006054, "minimum": 5.50365226395661, "sample_count": 5 }, "response_window_union_seconds": 27.70361144398339, "server_decode_tokens_per_second": { "maximum": 53.78570131386867, "mean": 53.413691716938814, "median": 53.41785224622068, "minimum": 53.055938852926445, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 0.7708376049995422, "mean": 0.7664965492091141, "median": 0.7665820000111125, "minimum": 0.7625974579714239, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 }, { "aggregate_output_tokens_per_second": 30.756055948328438, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1199176526486507, "interval_count": 5, "interval_seconds": 41.637717549921945, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.83736488475682, "process_cpu_seconds": 41.57 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 12482.987149010663, "reason": null, "response_union_seconds": 41.61781998805236, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9433212165913, "trapezoid_pieces": 46, "watt_hours": 3.4674964302807396 }, "gpu_sample_count": 41, "host_sample_count": 41, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17265410048, "mean": 17259049059.90244, "median": 17258135552, "minimum": 17254797312, "sample_count": 41 }, "gpu_board_watts": { "maximum": 300.29, "mean": 299.94853658536584, "median": 299.99, "minimum": 297.88, "sample_count": 41 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29347545088.0, "median": 29347545088.0, "minimum": 29347545088.0, "sample_count": 41 }, "gpu_sm_clock_mhz": { "maximum": 1665.0, "mean": 1610.9756097560976, "median": 1612.0, "minimum": 1590.0, "sample_count": 41 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.02439024390245, "median": 99.0, "minimum": 84.0, "sample_count": 41 }, "host_mem_available_bytes": { "maximum": 96122597376, "mean": 96096807361.56097, "median": 96098525184, "minimum": 96063385600, "sample_count": 41 }, "process_pss_bytes": { "maximum": 932377600, "mean": 927213143.4146341, "median": 926389248, "minimum": 924091392, "sample_count": 41 }, "process_rss_bytes": { "maximum": 935366656, "mean": 930202199.4146341, "median": 929378304, "minimum": 927080448, "sample_count": 41 } }, "process_sample_count": 41, "response_union_seconds": 41.61781998805236, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 20480 }, "observed_endpoint_seconds": 36.862388077017386, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 41.6480219400255, "response_seconds": { "maximum": 8.368119373975787, "mean": 8.323563997610472, "median": 8.326940262981225, "minimum": 8.274199456034694, "sample_count": 5 }, "response_window_union_seconds": 41.61781998805236, "server_decode_tokens_per_second": { "maximum": 51.768829236760986, "mean": 51.47410572541499, "median": 51.441366928244335, "minimum": 51.21463027254618, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 3.389060902001802, "mean": 3.369435553799849, "median": 3.36972534598317, "minimum": 3.3483261369983666, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 } ], "id": "concurrency-32b-c1", "model": "qwen3-32b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 417.3 }, { "device": "CUDA0", "reported_mib": 18423.65 } ], "gpu_layers": 65, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 65 }, "record_sha256": "e7c635b222262a77e32442c195329294cbb557d68ca7789b602c3ae5aaf23e22", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16922128384, "mean": 16913729536, "median": 16913729536.0, "minimum": 16905330688, "sample_count": 2 }, "gpu_board_watts": { "maximum": 16.67, "mean": 16.54, "median": 16.54, "minimum": 16.41, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96507301888, "mean": 96495747072, "median": 96495747072.0, "minimum": 96484192256, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1003568340092897, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16910635008, "mean": 16910635008, "median": 16910635008, "minimum": 16910635008, "sample_count": 1 }, "gpu_board_watts": { "maximum": 94.7, "mean": 94.7, "median": 94.7, "minimum": 94.7, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2302.0, "mean": 2302.0, "median": 2302.0, "minimum": 2302.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96518283264, "mean": 96518283264, "median": 96518283264, "minimum": 96518283264, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1621831920347176, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 229.67598352695532, "reason": null, "response_union_seconds": 3.5240373039850965, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 65.17410677441757, "trapezoid_pieces": 4, "watt_hours": 0.06379888431304315 }, "gpu_sample_count": 3, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17153597440, "mean": 17153525760, "median": 17153525760.0, "minimum": 17153454080, "sample_count": 2 }, "gpu_board_watts": { "maximum": 78.99, "mean": 70.92666666666666, "median": 78.93, "minimum": 54.86, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 19906166784.0, "mean": 19906166784.0, "median": 19906166784.0, "minimum": 19906166784.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2310.0, "mean": 2300.0, "median": 2310.0, "minimum": 2280.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 74.0, "mean": 74.0, "median": 74.0, "minimum": 74.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96197144576, "mean": 96193110016, "median": 96193110016.0, "minimum": 96189075456, "sample_count": 2 }, "process_pss_bytes": { "maximum": 20098963456, "mean": 20098891776, "median": 20098891776.0, "minimum": 20098820096, "sample_count": 2 }, "process_rss_bytes": { "maximum": 20101951488, "mean": 20101879808, "median": 20101879808.0, "minimum": 20101808128, "sample_count": 2 } }, "process_sample_count": 2, "response_union_seconds": 3.5240373039850965, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 5, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 1.2853985410183668, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 560.9414774929087, "reason": null, "response_union_seconds": 3.315524325997103, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 169.18635556209122, "trapezoid_pieces": 8, "watt_hours": 0.15581707708136353 }, "gpu_sample_count": 5, "host_sample_count": 4, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17265590272, "mean": 17234575360, "median": 17230043136.0, "minimum": 17212624896, "sample_count": 4 }, "gpu_board_watts": { "maximum": 299.4, "mean": 183.368, "median": 129.21, "minimum": 91.7, "sample_count": 5 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29342511923.2, "median": 29347545088.0, "minimum": 29322379264.0, "sample_count": 5 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2029.4, "median": 2310.0, "minimum": 1575.0, "sample_count": 5 }, "gpu_utilization_percent": { "maximum": 88.0, "mean": 31.2, "median": 0.0, "minimum": 0.0, "sample_count": 5 }, "host_mem_available_bytes": { "maximum": 96123850752, "mean": 96111056896, "median": 96119119872.0, "minimum": 96082137088, "sample_count": 4 }, "process_pss_bytes": { "maximum": 932377600, "mean": 901519872, "median": 897512960.0, "minimum": 878675968, "sample_count": 4 }, "process_rss_bytes": { "maximum": 935366656, "mean": 904508416, "median": 900501504.0, "minimum": 881664000, "sample_count": 4 } }, "process_sample_count": 4, "response_union_seconds": 3.315524325997103, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 1.0191477150074206, "observed_intervals": 1, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 20813.582241929886, "reason": null, "response_union_seconds": 69.39111728599528, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.94591607664665, "trapezoid_pieces": 70, "watt_hours": 5.781550622758302 }, "gpu_sample_count": 68, "host_sample_count": 68, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17265410048, "mean": 17253726268.235294, "median": 17256364032.0, "minimum": 17243848704, "sample_count": 68 }, "gpu_board_watts": { "maximum": 301.23, "mean": 299.98058823529414, "median": 299.99, "minimum": 297.88, "sample_count": 68 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29347545088.0, "median": 29347545088.0, "minimum": 29347545088.0, "sample_count": 68 }, "gpu_sm_clock_mhz": { "maximum": 1717.0, "mean": 1611.6176470588234, "median": 1605.0, "minimum": 1575.0, "sample_count": 68 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.08823529411765, "median": 99.0, "minimum": 84.0, "sample_count": 68 }, "host_mem_available_bytes": { "maximum": 96128315392, "mean": 96101368591.05882, "median": 96100413440.0, "minimum": 96063385600, "sample_count": 68 }, "process_pss_bytes": { "maximum": 932377600, "mean": 922582874.3529412, "median": 924959744.0, "minimum": 913896448, "sample_count": 68 }, "process_rss_bytes": { "maximum": 935366656, "mean": 925592696.4705882, "median": 927948800.0, "minimum": 917929984, "sample_count": 68 } }, "process_sample_count": 68, "response_union_seconds": 69.39111728599528, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 65536 }, "observed_endpoint_seconds": 67.61177103599766, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 3871.92128950453, "reason": null, "response_union_seconds": 13.758397663012147, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 281.4223999291531, "trapezoid_pieces": 16, "watt_hours": 1.075533691529036 }, "gpu_sample_count": 14, "host_sample_count": 13, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17253564416, "mean": 17248445991.384617, "median": 17248645120, "minimum": 17243058176, "sample_count": 13 }, "gpu_board_watts": { "maximum": 300.79, "mean": 285.5842857142857, "median": 299.995, "minimum": 128.26, "sample_count": 14 }, "gpu_memory_used_bytes": { "maximum": 29347545088.0, "mean": 29347545088.0, "median": 29347545088.0, "minimum": 29347545088.0, "sample_count": 14 }, "gpu_sm_clock_mhz": { "maximum": 1927.0, "mean": 1638.5, "median": 1623.5, "minimum": 1590.0, "sample_count": 14 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.28571428571429, "median": 99.0, "minimum": 99.0, "sample_count": 14 }, "host_mem_available_bytes": { "maximum": 96119128064, "mean": 96098498402.46153, "median": 96099074048, "minimum": 96077873152, "sample_count": 13 }, "process_pss_bytes": { "maximum": 923378688, "mean": 918623940.9230769, "median": 917521408, "minimum": 914600960, "sample_count": 13 }, "process_rss_bytes": { "maximum": 926367744, "mean": 921612918.1538461, "median": 920510464, "minimum": 917590016, "sample_count": 13 } }, "process_sample_count": 13, "response_union_seconds": 13.758397663012147, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 12288 }, "observed_endpoint_seconds": 11.25419971102383, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 1, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 93, "host_samples": 90, "maximum_gpu_gap_seconds": 1.0046468999935314, "maximum_host_gap_seconds": 1.2950242789811455, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 93, "unavailable": 0 }, "software": { "active": 0, "not_active": 93, "unavailable": 0 } } } }, { "case_wall_seconds": 420.7694540799712, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 55752.341995404386, "reason": null, "response_union_seconds": 185.9375148040126, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.84450450556, "trapezoid_pieces": 195, "watt_hours": 15.486761665390107 }, "gpu_sample_count": 185, "host_sample_count": 181, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17361494016, "mean": 17327632904.486187, "median": 17339219968, "minimum": 17279602688, "sample_count": 181 }, "gpu_board_watts": { "maximum": 300.9, "mean": 299.9730810810811, "median": 300.0, "minimum": 294.85, "sample_count": 185 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 185 }, "gpu_sm_clock_mhz": { "maximum": 1710.0, "mean": 1553.2648648648649, "median": 1537.0, "minimum": 1485.0, "sample_count": 185 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.14594594594594, "median": 99.0, "minimum": 74.0, "sample_count": 185 }, "host_mem_available_bytes": { "maximum": 96149176320, "mean": 96071103210.78453, "median": 96074928128, "minimum": 96005373952, "sample_count": 181 }, "process_pss_bytes": { "maximum": 997264384, "mean": 973108699.2265193, "median": 982801408, "minimum": 936352768, "sample_count": 181 }, "process_rss_bytes": { "maximum": 1000247296, "mean": 976107395.5359116, "median": 985784320, "minimum": 939335680, "sample_count": 181 } }, "process_sample_count": 181, "response_union_seconds": 185.9375148040126, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 188416 }, "observed_endpoint_seconds": 175.02184689906426, "observed_intervals": 10, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 81.84464957406752, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.12788657574322, "interval_count": 5, "interval_seconds": 62.57220179203432, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 100.09237042378304, "process_cpu_seconds": 62.629999999999995 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 18753.3813016355, "reason": null, "response_union_seconds": 62.55754073901335, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.7781095627398, "trapezoid_pieces": 67, "watt_hours": 5.209272583787639 }, "gpu_sample_count": 62, "host_sample_count": 61, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17298706432, "mean": 17287832693.508198, "median": 17288585216, "minimum": 17279602688, "sample_count": 61 }, "gpu_board_watts": { "maximum": 300.68, "mean": 299.9858064516129, "median": 300.01, "minimum": 299.23, "sample_count": 62 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 62 }, "gpu_sm_clock_mhz": { "maximum": 1665.0, "mean": 1549.6935483870968, "median": 1530.0, "minimum": 1485.0, "sample_count": 62 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.01612903225806, "median": 99.0, "minimum": 95.0, "sample_count": 62 }, "host_mem_available_bytes": { "maximum": 96080334848, "mean": 96040837120, "median": 96046120960, "minimum": 96005373952, "sample_count": 61 }, "process_pss_bytes": { "maximum": 948587520, "mean": 941003255.6065574, "median": 942095360, "minimum": 936352768, "sample_count": 61 }, "process_rss_bytes": { "maximum": 951570432, "mean": 944009417.4426229, "median": 945078272, "minimum": 939335680, "sample_count": 61 } }, "process_sample_count": 61, "response_union_seconds": 62.55754073901335, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 65536 }, "observed_endpoint_seconds": 57.27966735709924, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 62.60230736201629, "response_seconds": { "maximum": 12.592243108025286, "mean": 12.467036286104122, "median": 12.479891907511046, "minimum": 12.302360792993568, "sample_count": 20 }, "response_window_union_seconds": 62.55754073901335, "server_decode_tokens_per_second": { "maximum": 28.00551324221474, "mean": 25.58471676770163, "median": 26.37554127135465, "minimum": 21.76054778552292, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 3.3099877709755674, "mean": 2.412118052196456, "median": 2.786846547969617, "minimum": 0.7595875930273905, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 }, { "aggregate_output_tokens_per_second": 41.49782036185769, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.125263134343059, "interval_count": 5, "interval_seconds": 123.39960938395234, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 100.00842029897788, "process_cpu_seconds": 123.41 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 36998.96069376888, "reason": null, "response_union_seconds": 123.37997406499926, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.8781688370028, "trapezoid_pieces": 128, "watt_hours": 10.277489081602466 }, "gpu_sample_count": 123, "host_sample_count": 120, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17361494016, "mean": 17347864678.4, "median": 17350475776.0, "minimum": 17321365504, "sample_count": 120 }, "gpu_board_watts": { "maximum": 300.9, "mean": 299.96666666666664, "median": 299.99, "minimum": 294.85, "sample_count": 123 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 123 }, "gpu_sm_clock_mhz": { "maximum": 1710.0, "mean": 1555.0650406504064, "median": 1567.0, "minimum": 1500.0, "sample_count": 123 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.21138211382114, "median": 100.0, "minimum": 74.0, "sample_count": 123 }, "host_mem_available_bytes": { "maximum": 96149176320, "mean": 96086488473.6, "median": 96085280768.0, "minimum": 96021024768, "sample_count": 120 }, "process_pss_bytes": { "maximum": 997264384, "mean": 989428966.4, "median": 992402432.0, "minimum": 968424448, "sample_count": 120 }, "process_rss_bytes": { "maximum": 1000247296, "mean": 992423867.7333333, "median": 995385344.0, "minimum": 971407360, "sample_count": 120 } }, "process_sample_count": 120, "response_union_seconds": 123.37997406499926, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 122880 }, "observed_endpoint_seconds": 117.74217954196502, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 123.44470494805137, "response_seconds": { "maximum": 24.823961583024357, "mean": 24.52178838660766, "median": 24.525591763027478, "minimum": 24.108100534998812, "sample_count": 20 }, "response_window_union_seconds": 123.37997406499926, "server_decode_tokens_per_second": { "maximum": 24.761002915778022, "mean": 18.143512774672224, "median": 17.788452280236555, "minimum": 12.163616381481537, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 14.377657943987288, "mean": 9.506834106854512, "median": 10.070695419010008, "minimum": 3.423168073000852, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 } ], "id": "concurrency-32b-c4", "model": "qwen3-32b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 417.3 }, { "device": "CUDA0", "reported_mib": 18423.65 } ], "gpu_layers": 65, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 65 }, "record_sha256": "806238b5092d95f97811d67523489974a389b64477b4f6fbfabd0f3de1d3e257", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16936005632, "mean": 16927412224, "median": 16927412224.0, "minimum": 16918818816, "sample_count": 2 }, "gpu_board_watts": { "maximum": 15.99, "mean": 15.905000000000001, "median": 15.905000000000001, "minimum": 15.82, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96495370240, "mean": 96480976896, "median": 96480976896.0, "minimum": 96466583552, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1003576699877158, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16943104000, "mean": 16943104000, "median": 16943104000, "minimum": 16943104000, "sample_count": 1 }, "gpu_board_watts": { "maximum": 105.94, "mean": 105.94, "median": 105.94, "minimum": 105.94, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2340.0, "median": 2340.0, "minimum": 2340.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96572526592, "mean": 96572526592, "median": 96572526592, "minimum": 96572526592, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1639506969950162, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 207.79273563988033, "reason": null, "response_union_seconds": 3.5026424899697304, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 59.324563164788216, "trapezoid_pieces": 4, "watt_hours": 0.057720204344411205 }, "gpu_sample_count": 3, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17226100736, "mean": 17186957994.666668, "median": 17167605760, "minimum": 17167167488, "sample_count": 3 }, "gpu_board_watts": { "maximum": 78.3, "mean": 64.33666666666667, "median": 78.19, "minimum": 36.52, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 19906166784.0, "mean": 19906166784.0, "median": 19906166784.0, "minimum": 19906166784.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2310.0, "mean": 2300.0, "median": 2310.0, "minimum": 2280.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 78.0, "mean": 78.0, "median": 78.0, "minimum": 78.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96178794496, "mean": 96163105450.66667, "median": 96178565120, "minimum": 96131956736, "sample_count": 3 }, "process_pss_bytes": { "maximum": 20098980864, "mean": 13692147370.666666, "median": 20098866176, "minimum": 878595072, "sample_count": 3 }, "process_rss_bytes": { "maximum": 20101963776, "mean": 13695130282.666666, "median": 20101849088, "minimum": 881577984, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.5026424899697304, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 13, "storage_read_bytes": 0, "storage_write_bytes": 4096 }, "observed_endpoint_seconds": 2.2705488849896938, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 575.8172122131975, "reason": null, "response_union_seconds": 3.3165422920137644, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 173.6197405351247, "trapezoid_pieces": 6, "watt_hours": 0.1599492256147771 }, "gpu_sample_count": 3, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17363476480, "mean": 17296278869.333332, "median": 17299456000, "minimum": 17225904128, "sample_count": 3 }, "gpu_board_watts": { "maximum": 241.87, "mean": 186.64666666666668, "median": 227.52, "minimum": 90.55, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29340554581.333332, "median": 29349642240.0, "minimum": 29322379264.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2327.3333333333335, "median": 2332.0, "minimum": 2310.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96134615040, "mean": 96089931776, "median": 96077000704, "minimum": 96058179584, "sample_count": 3 }, "process_pss_bytes": { "maximum": 997264384, "mean": 941612032, "median": 948870144, "minimum": 878701568, "sample_count": 3 }, "process_rss_bytes": { "maximum": 1000247296, "mean": 944594944, "median": 951853056, "minimum": 881684480, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.3165422920137644, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 55792.14584703776, "reason": null, "response_union_seconds": 186.07392277405597, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.8386072334515, "trapezoid_pieces": 187, "watt_hours": 15.497818290843822 }, "gpu_sample_count": 185, "host_sample_count": 181, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17361494016, "mean": 17327632904.486187, "median": 17339219968, "minimum": 17279602688, "sample_count": 181 }, "gpu_board_watts": { "maximum": 300.9, "mean": 299.9730810810811, "median": 300.0, "minimum": 294.85, "sample_count": 185 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 185 }, "gpu_sm_clock_mhz": { "maximum": 1710.0, "mean": 1553.2648648648649, "median": 1537.0, "minimum": 1485.0, "sample_count": 185 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.14594594594594, "median": 99.0, "minimum": 74.0, "sample_count": 185 }, "host_mem_available_bytes": { "maximum": 96149176320, "mean": 96071103210.78453, "median": 96074928128, "minimum": 96005373952, "sample_count": 181 }, "process_pss_bytes": { "maximum": 997264384, "mean": 973108699.2265193, "median": 982801408, "minimum": 936352768, "sample_count": 181 }, "process_rss_bytes": { "maximum": 1000247296, "mean": 976107395.5359116, "median": 985784320, "minimum": 939335680, "sample_count": 181 } }, "process_sample_count": 181, "response_union_seconds": 186.07392277405597, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 229376 }, "observed_endpoint_seconds": 183.24035443703178, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 10747.853438441543, "reason": null, "response_union_seconds": 36.686159758944996, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 292.96752533006537, "trapezoid_pieces": 40, "watt_hours": 2.98551484401154 }, "gpu_sample_count": 38, "host_sample_count": 36, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17317265408, "mean": 17296877340.444443, "median": 17303889920.0, "minimum": 17255440384, "sample_count": 36 }, "gpu_board_watts": { "maximum": 300.91, "mean": 291.04894736842107, "median": 299.98, "minimum": 127.66, "sample_count": 38 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349476675.36842, "median": 29349642240.0, "minimum": 29347545088.0, "sample_count": 38 }, "gpu_sm_clock_mhz": { "maximum": 1965.0, "mean": 1588.0263157894738, "median": 1586.0, "minimum": 1515.0, "sample_count": 38 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 98.92105263157895, "median": 99.0, "minimum": 94.0, "sample_count": 38 }, "host_mem_available_bytes": { "maximum": 96110641152, "mean": 96048308565.33333, "median": 96044918784.0, "minimum": 96014688256, "sample_count": 36 }, "process_pss_bytes": { "maximum": 965807104, "mean": 948997688.8888888, "median": 953592832.0, "minimum": 912342016, "sample_count": 36 }, "process_rss_bytes": { "maximum": 968790016, "mean": 951980600.8888888, "median": 956575744.0, "minimum": 915324928, "sample_count": 36 } }, "process_sample_count": 36, "response_union_seconds": 36.686159758944996, "sampled_counter_deltas": { "deltas": { "major_faults": 6, "read_syscall_bytes": 0, "storage_read_bytes": 1179648, "storage_write_bytes": 40960 }, "observed_endpoint_seconds": 34.84366085496731, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 4, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 232, "host_samples": 226, "maximum_gpu_gap_seconds": 1.0094260819605552, "maximum_host_gap_seconds": 1.306493810028769, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 232, "unavailable": 0 }, "software": { "active": 0, "not_active": 232, "unavailable": 0 } } } }, { "case_wall_seconds": 285.04197112697875, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 43583.755059872135, "reason": null, "response_union_seconds": 145.35013979504583, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.8535475874214, "trapezoid_pieces": 155, "watt_hours": 12.10659862774226 }, "gpu_sample_count": 145, "host_sample_count": 142, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17379631104, "mean": 17357138727.661972, "median": 17368823808.0, "minimum": 17327955968, "sample_count": 142 }, "gpu_board_watts": { "maximum": 303.17, "mean": 299.96124137931037, "median": 299.99, "minimum": 296.54, "sample_count": 145 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 145 }, "gpu_sm_clock_mhz": { "maximum": 1627.0, "mean": 1478.8689655172413, "median": 1485.0, "minimum": 1305.0, "sample_count": 145 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.64137931034483, "median": 100.0, "minimum": 83.0, "sample_count": 145 }, "host_mem_available_bytes": { "maximum": 96155914240, "mean": 96071287735.88733, "median": 96036245504.0, "minimum": 96016179200, "sample_count": 142 }, "process_pss_bytes": { "maximum": 1253164032, "mean": 1233515772.3943663, "median": 1245466624.0, "minimum": 1207312384, "sample_count": 142 }, "process_rss_bytes": { "maximum": 1256132608, "mean": 1236515392.9014084, "median": 1248436224.0, "minimum": 1210281984, "sample_count": 142 } }, "process_sample_count": 142, "response_union_seconds": 145.35013979504583, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 102400 }, "observed_endpoint_seconds": 135.45640823506983, "observed_intervals": 10, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 22.028412284302284, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1210980722289743, "interval_count": 5, "interval_seconds": 58.12257282593055, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.87513831132718, "process_cpu_seconds": 58.05 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 17424.8135125459, "reason": null, "response_union_seconds": 58.10677517199656, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.8757625245645, "trapezoid_pieces": 63, "watt_hours": 4.840225975707194 }, "gpu_sample_count": 58, "host_sample_count": 57, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17349152768, "mean": 17331692166.736843, "median": 17331625984, "minimum": 17327955968, "sample_count": 57 }, "gpu_board_watts": { "maximum": 300.28, "mean": 299.96224137931034, "median": 299.99, "minimum": 298.49, "sample_count": 58 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 58 }, "gpu_sm_clock_mhz": { "maximum": 1627.0, "mean": 1451.5172413793102, "median": 1485.0, "minimum": 1305.0, "sample_count": 58 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.70689655172414, "median": 100.0, "minimum": 86.0, "sample_count": 58 }, "host_mem_available_bytes": { "maximum": 96047636480, "mean": 96030269511.85965, "median": 96029605888, "minimum": 96017846272, "sample_count": 57 }, "process_pss_bytes": { "maximum": 1218039808, "mean": 1209269230.0350878, "median": 1209229312, "minimum": 1207312384, "sample_count": 57 }, "process_rss_bytes": { "maximum": 1221009408, "mean": 1212263729.4035087, "median": 1212198912, "minimum": 1210281984, "sample_count": 57 } }, "process_sample_count": 57, "response_union_seconds": 58.10677517199656, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 36864 }, "observed_endpoint_seconds": 53.336369540018495, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 58.13016001897631, "response_seconds": { "maximum": 11.71224289300153, "mean": 11.621355034399311, "median": 11.628675386018585, "minimum": 11.497684480971657, "sample_count": 5 }, "response_window_union_seconds": 58.10677517199656, "server_decode_tokens_per_second": { "maximum": 25.931278654062915, "mean": 25.626653456527215, "median": 25.57785387161442, "minimum": 25.41033708456341, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 1.6838101929752156, "mean": 1.670307478599716, "median": 1.6676911130198278, "minimum": 1.6591309130308218, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 }, { "aggregate_output_tokens_per_second": 14.671602883846484, "completed_output_tokens": 1280, "completed_request_count": 5, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.120939686796973, "interval_count": 5, "interval_seconds": 87.26338133099489, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.87006997750314, "process_cpu_seconds": 87.15 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 26158.941547326252, "reason": null, "response_union_seconds": 87.24336462304927, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.83875175322146, "trapezoid_pieces": 92, "watt_hours": 7.26637265203507 }, "gpu_sample_count": 87, "host_sample_count": 85, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17379631104, "mean": 17374202892.04706, "median": 17376063488, "minimum": 17363415040, "sample_count": 85 }, "gpu_board_watts": { "maximum": 303.17, "mean": 299.9605747126437, "median": 300.0, "minimum": 296.54, "sample_count": 87 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 87 }, "gpu_sm_clock_mhz": { "maximum": 1545.0, "mean": 1497.103448275862, "median": 1477.0, "minimum": 1462.0, "sample_count": 87 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.59770114942529, "median": 100.0, "minimum": 83.0, "sample_count": 87 }, "host_mem_available_bytes": { "maximum": 96155914240, "mean": 96098794074.35294, "median": 96110546944, "minimum": 96016179200, "sample_count": 85 }, "process_pss_bytes": { "maximum": 1253164032, "mean": 1249775218.447059, "median": 1251803136, "minimum": 1240936448, "sample_count": 85 }, "process_rss_bytes": { "maximum": 1256132608, "mean": 1252778273.1294117, "median": 1254772736, "minimum": 1243906048, "sample_count": 85 } }, "process_sample_count": 85, "response_union_seconds": 87.24336462304927, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 65536 }, "observed_endpoint_seconds": 82.12003869505133, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 87.273929255025, "response_seconds": { "maximum": 17.565991370007396, "mean": 17.448672924609856, "median": 17.456415257009212, "minimum": 17.328929729992524, "sample_count": 5 }, "response_window_union_seconds": 87.24336462304927, "server_decode_tokens_per_second": { "maximum": 24.89443782111384, "mean": 24.72205583310525, "median": 24.709477111611353, "minimum": 24.551998243810004, "sample_count": 5 }, "synthetic_first_token_seconds": { "maximum": 7.1797569649643265, "mean": 7.133594566385727, "median": 7.136343034973834, "minimum": 7.085526316019241, "sample_count": 5 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 1 } ], "id": "concurrency-72b-q4-c1", "model": "qwen25-72b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 668.25 }, { "device": "CUDA0", "reported_mib": 41297.69 } ], "gpu_layers": 81, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 81 }, "record_sha256": "c9243a2bc147bb4e2eacacfe6db0e0f45bfb9e1ff382df0e73f4001e22529e1d", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16978206720, "mean": 16969136128, "median": 16969136128.0, "minimum": 16960065536, "sample_count": 2 }, "gpu_board_watts": { "maximum": 16.06, "mean": 15.985, "median": 15.985, "minimum": 15.91, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96520728576, "mean": 96520091648, "median": 96520091648.0, "minimum": 96519454720, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1008190379943699, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16965869568, "mean": 16965869568, "median": 16965869568, "minimum": 16965869568, "sample_count": 1 }, "gpu_board_watts": { "maximum": 106.5, "mean": 106.5, "median": 106.5, "minimum": 106.5, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2340.0, "median": 2340.0, "minimum": 2340.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96598876160, "mean": 96598876160, "median": 96598876160, "minimum": 96598876160, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.159519933979027, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 500.40460333691567, "reason": null, "response_union_seconds": 7.142398557043634, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 70.06114253361437, "trapezoid_pieces": 8, "watt_hours": 0.1390012787046988 }, "gpu_sample_count": 7, "host_sample_count": 5, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17257222144, "mean": 17253466931.2, "median": 17256321024, "minimum": 17242816512, "sample_count": 5 }, "gpu_board_watts": { "maximum": 79.11, "mean": 73.61285714285714, "median": 78.44, "minimum": 46.09, "sample_count": 7 }, "gpu_memory_used_bytes": { "maximum": 55622762496.0, "mean": 39380919442.28571, "median": 43891294208.0, "minimum": 587202560.0, "sample_count": 7 }, "gpu_sm_clock_mhz": { "maximum": 2310.0, "mean": 2299.0, "median": 2302.0, "minimum": 2272.0, "sample_count": 7 }, "gpu_utilization_percent": { "maximum": 78.0, "mean": 57.57142857142857, "median": 76.0, "minimum": 0.0, "sample_count": 7 }, "host_mem_available_bytes": { "maximum": 96174260224, "mean": 96142704640, "median": 96146825216, "minimum": 96110252032, "sample_count": 5 }, "process_pss_bytes": { "maximum": 44347047936, "mean": 34410967040, "median": 44346761216, "minimum": 1126739968, "sample_count": 5 }, "process_rss_bytes": { "maximum": 44350017536, "mean": 34413936640, "median": 44349730816, "minimum": 1129709568, "sample_count": 5 } }, "process_sample_count": 5, "response_union_seconds": 7.142398557043634, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 13, "storage_read_bytes": 0, "storage_write_bytes": 8192 }, "observed_endpoint_seconds": 5.804069469973911, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 570.0591014654404, "reason": null, "response_union_seconds": 3.319998638005927, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 171.7046190741307, "trapezoid_pieces": 6, "watt_hours": 0.15834975040706678 }, "gpu_sample_count": 3, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17379704832, "mean": 17336277674.666668, "median": 17340379136, "minimum": 17288749056, "sample_count": 3 }, "gpu_board_watts": { "maximum": 286.45, "mean": 199.11666666666667, "median": 209.48, "minimum": 101.42, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55650025472.0, "median": 55660511232.0, "minimum": 55629053952.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2127.3333333333335, "median": 2310.0, "minimum": 1732.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 20.0, "mean": 6.666666666666667, "median": 0.0, "minimum": 0.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96096833536, "mean": 96065088170.66667, "median": 96067203072, "minimum": 96031227904, "sample_count": 3 }, "process_pss_bytes": { "maximum": 1253164032, "mean": 1210951338.6666667, "median": 1218109440, "minimum": 1161580544, "sample_count": 3 }, "process_rss_bytes": { "maximum": 1256132608, "mean": 1213920597.3333333, "median": 1221079040, "minimum": 1164550144, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.319998638005927, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 43603.333030460024, "reason": null, "response_union_seconds": 145.41683486895636, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.85065394769146, "trapezoid_pieces": 147, "watt_hours": 12.112036952905562 }, "gpu_sample_count": 145, "host_sample_count": 142, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17379631104, "mean": 17357138727.661972, "median": 17368823808.0, "minimum": 17327955968, "sample_count": 142 }, "gpu_board_watts": { "maximum": 303.17, "mean": 299.96124137931037, "median": 299.99, "minimum": 296.54, "sample_count": 145 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 145 }, "gpu_sm_clock_mhz": { "maximum": 1627.0, "mean": 1478.8689655172413, "median": 1485.0, "minimum": 1305.0, "sample_count": 145 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.64137931034483, "median": 100.0, "minimum": 83.0, "sample_count": 145 }, "host_mem_available_bytes": { "maximum": 96155914240, "mean": 96071287735.88733, "median": 96036245504.0, "minimum": 96016179200, "sample_count": 142 }, "process_pss_bytes": { "maximum": 1253164032, "mean": 1233515772.3943663, "median": 1245466624.0, "minimum": 1207312384, "sample_count": 142 }, "process_rss_bytes": { "maximum": 1256132608, "mean": 1236515392.9014084, "median": 1248436224.0, "minimum": 1210281984, "sample_count": 142 } }, "process_sample_count": 142, "response_union_seconds": 145.41683486895636, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 126976 }, "observed_endpoint_seconds": 143.68809728400083, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 8338.197633615331, "reason": null, "response_union_seconds": 28.63761988701299, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 291.1623824365607, "trapezoid_pieces": 31, "watt_hours": 2.316166009337592 }, "gpu_sample_count": 29, "host_sample_count": 28, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17373618176, "mean": 17343057334.857143, "median": 17341583360.0, "minimum": 17320845312, "sample_count": 28 }, "gpu_board_watts": { "maximum": 300.15, "mean": 289.5420689655172, "median": 300.0, "minimum": 115.61, "sample_count": 29 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 29 }, "gpu_sm_clock_mhz": { "maximum": 2332.0, "mean": 1529.0344827586207, "median": 1485.0, "minimum": 1440.0, "sample_count": 29 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 96.62068965517241, "median": 100.0, "minimum": 2.0, "sample_count": 29 }, "host_mem_available_bytes": { "maximum": 96039104512, "mean": 96025997019.42857, "median": 96028440576.0, "minimum": 96003043328, "sample_count": 28 }, "process_pss_bytes": { "maximum": 1240260608, "mean": 1219563081.142857, "median": 1218818048.0, "minimum": 1198239744, "sample_count": 28 }, "process_rss_bytes": { "maximum": 1243230208, "mean": 1222634057.142857, "median": 1221787648.0, "minimum": 1201209344, "sample_count": 28 } }, "process_sample_count": 28, "response_union_seconds": 28.63761988701299, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 24576 }, "observed_endpoint_seconds": 26.66104468901176, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 1, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 187, "host_samples": 181, "maximum_gpu_gap_seconds": 1.0082573510007933, "maximum_host_gap_seconds": 1.6206080699921586, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 187, "unavailable": 0 }, "software": { "active": 0, "not_active": 187, "unavailable": 0 } } } }, { "case_wall_seconds": 719.4450020020013, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 116153.30866974915, "reason": null, "response_union_seconds": 387.2295688489685, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9598119921791, "trapezoid_pieces": 397, "watt_hours": 32.26480796381921 }, "gpu_sample_count": 387, "host_sample_count": 376, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17462124544, "mean": 17423276674.723404, "median": 17432784896.0, "minimum": 17366061056, "sample_count": 376 }, "gpu_board_watts": { "maximum": 301.05, "mean": 299.9867958656331, "median": 299.99, "minimum": 298.38, "sample_count": 387 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 387 }, "gpu_sm_clock_mhz": { "maximum": 1635.0, "mean": 1438.095607235142, "median": 1417.0, "minimum": 1342.0, "sample_count": 387 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.6795865633075, "median": 100.0, "minimum": 91.0, "sample_count": 387 }, "host_mem_available_bytes": { "maximum": 96145973248, "mean": 96063168871.48936, "median": 96063717376.0, "minimum": 96005341184, "sample_count": 376 }, "process_pss_bytes": { "maximum": 1295631360, "mean": 1278426348.93617, "median": 1287165440.0, "minimum": 1231869952, "sample_count": 376 }, "process_rss_bytes": { "maximum": 1298599936, "mean": 1281417673.531915, "median": 1290297344.0, "minimum": 1236250624, "sample_count": 376 } }, "process_sample_count": 376, "response_union_seconds": 387.2295688489685, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 331776 }, "observed_endpoint_seconds": 375.2016968349926, "observed_intervals": 10, "requested_intervals": 10 } }, "groups": [ { "aggregate_output_tokens_per_second": 38.93995189858778, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 2048, "cpu_snapshot_intervals": { "host_capacity_percent": 3.124980218163133, "interval_count": 5, "interval_seconds": 131.50083242496476, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.99936698122026, "process_cpu_seconds": 131.5 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 39432.121308624584, "reason": null, "response_union_seconds": 131.484497293015, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.8993959017812, "trapezoid_pieces": 136, "watt_hours": 10.953367030173496 }, "gpu_sample_count": 131, "host_sample_count": 127, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17425453056, "mean": 17393815616.503937, "median": 17401016320, "minimum": 17366061056, "sample_count": 127 }, "gpu_board_watts": { "maximum": 300.33, "mean": 299.9786259541985, "median": 299.99, "minimum": 298.61, "sample_count": 131 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 131 }, "gpu_sm_clock_mhz": { "maximum": 1582.0, "mean": 1428.8091603053435, "median": 1395.0, "minimum": 1357.0, "sample_count": 131 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.58778625954199, "median": 100.0, "minimum": 93.0, "sample_count": 131 }, "host_mem_available_bytes": { "maximum": 96145973248, "mean": 96061832957.98425, "median": 96068395008, "minimum": 96005341184, "sample_count": 127 }, "process_pss_bytes": { "maximum": 1273496576, "mean": 1257087443.6535432, "median": 1265202176, "minimum": 1231869952, "sample_count": 127 }, "process_rss_bytes": { "maximum": 1276465152, "mean": 1260089569.7637796, "median": 1268170752, "minimum": 1236250624, "sample_count": 127 } }, "process_sample_count": 127, "response_union_seconds": 131.484497293015, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 102400 }, "observed_endpoint_seconds": 125.07762075704522, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 131.53245825797785, "response_seconds": { "maximum": 26.65054602897726, "mean": 26.200603126001077, "median": 26.24883686102112, "minimum": 25.647156226972584, "sample_count": 20 }, "response_window_union_seconds": 131.484497293015, "server_decode_tokens_per_second": { "maximum": 13.635842265924056, "mean": 12.298470100174935, "median": 12.722440495557054, "minimum": 10.334289545557116, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 7.21949907601811, "mean": 5.264546025349409, "median": 6.091464050492505, "minimum": 1.6687007790314965, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 }, { "aggregate_output_tokens_per_second": 20.01993613738052, "completed_output_tokens": 5120, "completed_request_count": 20, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1235835368990346, "interval_count": 5, "interval_seconds": 255.76593056099955, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 99.9546731807691, "process_cpu_seconds": 255.64999999999998 }, "failed_request_count": 0, "measured_rounds": 5, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 76721.1873611245, "reason": null, "response_union_seconds": 255.74507155595347, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.99087331127305, "trapezoid_pieces": 261, "watt_hours": 21.31144093364569 }, "gpu_sample_count": 256, "host_sample_count": 249, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17462124544, "mean": 17438302997.590363, "median": 17439318016, "minimum": 17421230080, "sample_count": 249 }, "gpu_board_watts": { "maximum": 301.05, "mean": 299.9909765625, "median": 300.0, "minimum": 298.38, "sample_count": 256 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 256 }, "gpu_sm_clock_mhz": { "maximum": 1635.0, "mean": 1442.84765625, "median": 1500.0, "minimum": 1342.0, "sample_count": 256 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.7265625, "median": 100.0, "minimum": 91.0, "sample_count": 256 }, "host_mem_available_bytes": { "maximum": 96117440512, "mean": 96063850241.0281, "median": 96062423040, "minimum": 96024403968, "sample_count": 249 }, "process_pss_bytes": { "maximum": 1295631360, "mean": 1289310047.6144578, "median": 1290777600, "minimum": 1279087616, "sample_count": 249 }, "process_rss_bytes": { "maximum": 1298599936, "mean": 1292295863.0040162, "median": 1293746176, "minimum": 1282056192, "sample_count": 249 } }, "process_sample_count": 249, "response_union_seconds": 255.74507155595347, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 229376 }, "observed_endpoint_seconds": 250.1240760779474, "observed_intervals": 5, "requested_intervals": 5 } }, "response_outer_span_seconds": 255.81028590694768, "response_seconds": { "maximum": 51.4202549259644, "mean": 50.847095357597574, "median": 50.86070125648985, "minimum": 50.162616960995365, "sample_count": 20 }, "response_window_union_seconds": 255.74507155595347, "server_decode_tokens_per_second": { "maximum": 12.221575730155275, "mean": 8.901997778536677, "median": 8.665635784306376, "minimum": 5.8851670963954135, "sample_count": 20 }, "synthetic_first_token_seconds": { "maximum": 30.19842599896947, "mean": 20.22669281999115, "median": 21.229655119474046, "minimum": 7.263917299977038, "sample_count": 20 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 } ], "id": "concurrency-72b-q4-c4", "model": "qwen25-72b-q4", "module": "concurrency", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 668.25 }, { "device": "CUDA0", "reported_mib": 41297.69 } ], "gpu_layers": 81, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 81 }, "record_sha256": "6b5e6972abf6be1b6a331c2f2a22b580fb83bd46502a421d8507dfd60ae28554", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16989462528, "mean": 16980873216, "median": 16980873216.0, "minimum": 16972283904, "sample_count": 2 }, "gpu_board_watts": { "maximum": 15.82, "mean": 15.77, "median": 15.77, "minimum": 15.72, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96533319680, "mean": 96521064448, "median": 96521064448.0, "minimum": 96508809216, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1004123980528675, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 1, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 16997015552, "mean": 16997015552, "median": 16997015552, "minimum": 16997015552, "sample_count": 1 }, "gpu_board_watts": { "maximum": 112.0, "mean": 112.0, "median": 112.0, "minimum": 112.0, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2340.0, "median": 2340.0, "minimum": 2340.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 96592494592, "mean": 96592494592, "median": 96592494592, "minimum": 96592494592, "sample_count": 1 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1598984710290097, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 500.6619383217032, "reason": null, "response_union_seconds": 7.096872266964056, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 70.54684366411452, "trapezoid_pieces": 8, "watt_hours": 0.13907276064491755 }, "gpu_sample_count": 7, "host_sample_count": 5, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17269374976, "mean": 17249575731.2, "median": 17268948992, "minimum": 17184108544, "sample_count": 5 }, "gpu_board_watts": { "maximum": 78.69, "mean": 74.09714285714286, "median": 78.52, "minimum": 50.66, "sample_count": 7 }, "gpu_memory_used_bytes": { "maximum": 55622762496.0, "mean": 39380919442.28571, "median": 43891294208.0, "minimum": 587202560.0, "sample_count": 7 }, "gpu_sm_clock_mhz": { "maximum": 2310.0, "mean": 2300.0, "median": 2302.0, "minimum": 2272.0, "sample_count": 7 }, "gpu_utilization_percent": { "maximum": 81.0, "mean": 57.714285714285715, "median": 78.0, "minimum": 0.0, "sample_count": 7 }, "host_mem_available_bytes": { "maximum": 96162697216, "mean": 96149157478.4, "median": 96146513920, "minimum": 96130719744, "sample_count": 5 }, "process_pss_bytes": { "maximum": 44347048960, "mean": 34483454976, "median": 44346774528, "minimum": 1079759872, "sample_count": 5 }, "process_rss_bytes": { "maximum": 44350017536, "mean": 34486423552, "median": 44349743104, "minimum": 1082728448, "sample_count": 5 } }, "process_sample_count": 5, "response_union_seconds": 7.096872266964056, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 5, "storage_read_bytes": 0, "storage_write_bytes": 8192 }, "observed_endpoint_seconds": 5.7938264219556, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 582.8318020362483, "reason": null, "response_union_seconds": 3.319325248012319, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 175.5874337367994, "trapezoid_pieces": 6, "watt_hours": 0.16189772278784675 }, "gpu_sample_count": 3, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17452093440, "mean": 17394784938.666668, "median": 17414164480, "minimum": 17318096896, "sample_count": 3 }, "gpu_board_watts": { "maximum": 255.29, "mean": 168.00333333333333, "median": 145.64, "minimum": 103.08, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55650025472.0, "median": 55660511232.0, "minimum": 55629053952.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2160.0, "median": 2310.0, "minimum": 1830.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96079769600, "mean": 96071659520, "median": 96077111296, "minimum": 96058097664, "sample_count": 3 }, "process_pss_bytes": { "maximum": 1295631360, "mean": 1243190954.6666667, "median": 1273775104, "minimum": 1160166400, "sample_count": 3 }, "process_rss_bytes": { "maximum": 1298599936, "mean": 1246632618.6666667, "median": 1276743680, "minimum": 1164554240, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 3.319325248012319, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 3 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 116195.08796807335, "reason": null, "response_union_seconds": 387.37040795898065, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9586070094891, "trapezoid_pieces": 389, "watt_hours": 32.27641332446482 }, "gpu_sample_count": 387, "host_sample_count": 377, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17462124544, "mean": 17423226070.57825, "median": 17432780800, "minimum": 17366061056, "sample_count": 377 }, "gpu_board_watts": { "maximum": 301.05, "mean": 299.9867958656331, "median": 299.99, "minimum": 298.38, "sample_count": 387 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 387 }, "gpu_sm_clock_mhz": { "maximum": 1635.0, "mean": 1438.095607235142, "median": 1417.0, "minimum": 1342.0, "sample_count": 387 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.6795865633075, "median": 100.0, "minimum": 91.0, "sample_count": 387 }, "host_mem_available_bytes": { "maximum": 96145973248, "mean": 96063112493.49602, "median": 96063148032, "minimum": 96005341184, "sample_count": 377 }, "process_pss_bytes": { "maximum": 1295631360, "mean": 1278397334.0689654, "median": 1287002112, "minimum": 1231869952, "sample_count": 377 }, "process_rss_bytes": { "maximum": 1298599936, "mean": 1281388598.3236074, "median": 1290297344, "minimum": 1236250624, "sample_count": 377 } }, "process_sample_count": 377, "response_union_seconds": 387.37040795898065, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 372736 }, "observed_endpoint_seconds": 384.5076979199657, "observed_intervals": 2, "requested_intervals": 2 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 22950.795180775483, "reason": null, "response_union_seconds": 77.29993101599393, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 296.9057653625434, "trapezoid_pieces": 80, "watt_hours": 6.375220883548746 }, "gpu_sample_count": 78, "host_sample_count": 76, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 17433354240, "mean": 17396034937.263157, "median": 17416282112.0, "minimum": 17334210560, "sample_count": 76 }, "gpu_board_watts": { "maximum": 306.29, "mean": 296.84, "median": 299.99, "minimum": 129.61, "sample_count": 78 }, "gpu_memory_used_bytes": { "maximum": 55660511232.0, "mean": 55660511232.0, "median": 55660511232.0, "minimum": 55660511232.0, "sample_count": 78 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 1446.8589743589744, "median": 1380.0, "minimum": 1297.0, "sample_count": 78 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.76923076923077, "median": 100.0, "minimum": 98.0, "sample_count": 78 }, "host_mem_available_bytes": { "maximum": 96131813376, "mean": 96065721559.57895, "median": 96062631936.0, "minimum": 96013639680, "sample_count": 76 }, "process_pss_bytes": { "maximum": 1276486656, "mean": 1258132938.1052632, "median": 1276478464.0, "minimum": 1198920704, "sample_count": 76 }, "process_rss_bytes": { "maximum": 1279455232, "mean": 1261101918.3157895, "median": 1279447040.0, "minimum": 1201889280, "sample_count": 76 } }, "process_sample_count": 76, "response_union_seconds": 77.29993101599393, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 65536 }, "observed_endpoint_seconds": 75.86656208103523, "observed_intervals": 2, "requested_intervals": 2 } } }, "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 4, "context_shift": false, "contexts": [ 2048, 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": 5, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 478, "host_samples": 464, "maximum_gpu_gap_seconds": 1.0062838830053806, "maximum_host_gap_seconds": 1.6154293110012077, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 478, "unavailable": 0 }, "software": { "active": 0, "not_active": 478, "unavailable": 0 } } } }, { "case_wall_seconds": 14450.58151647501, "completed_response_resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 4322526.512667371, "reason": null, "response_union_seconds": 14409.160561117635, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9846170311602, "trapezoid_pieces": 14984, "watt_hours": 1200.7018090742697 }, "gpu_sample_count": 14408, "host_sample_count": 14044, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 7715704832, "mean": 6801911819.374537, "median": 6799910912.0, "minimum": 5833928704, "sample_count": 14044 }, "gpu_board_watts": { "maximum": 305.13, "mean": 299.9847897001666, "median": 299.99, "minimum": 283.16, "sample_count": 14408 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 14408 }, "gpu_sm_clock_mhz": { "maximum": 1792.0, "mean": 1531.5502498611881, "median": 1545.0, "minimum": 1477.0, "sample_count": 14408 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.21002220988339, "median": 100.0, "minimum": 50.0, "sample_count": 14408 }, "host_mem_available_bytes": { "maximum": 96192446464, "mean": 95707652121.37396, "median": 95703994368.0, "minimum": 95185977344, "sample_count": 14044 }, "process_pss_bytes": { "maximum": 1000995840, "mean": 994586522.5041299, "median": 999561216.0, "minimum": 939293696, "sample_count": 14044 }, "process_rss_bytes": { "maximum": 1003978752, "mean": 997597475.3631444, "median": 1002545152.0, "minimum": 942276608, "sample_count": 14044 } }, "process_sample_count": 14044, "response_union_seconds": 14409.160561117635, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 14311424 }, "observed_endpoint_seconds": 13820.278715579421, "observed_intervals": 576, "requested_intervals": 576 } }, "groups": [ { "aggregate_output_tokens_per_second": 40.933959858259136, "completed_output_tokens": 589824, "completed_request_count": 2304, "context_tokens": 8192, "cpu_snapshot_intervals": { "host_capacity_percent": 3.1251006357900364, "interval_count": 576, "interval_seconds": 14411.525899105764, "meaning": "Weighted CPU counter rate across round-bracketing snapshots, not exact token time.", "one_logical_cpu_percent": 100.00322034528116, "process_cpu_seconds": 14411.99 }, "failed_request_count": 0, "measured_rounds": 576, "mode": "completion", "resources": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 4322526.512667371, "reason": null, "response_union_seconds": 14409.160561117635, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.9846170311602, "trapezoid_pieces": 14984, "watt_hours": 1200.7018090742697 }, "gpu_sample_count": 14408, "host_sample_count": 14044, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 7715704832, "mean": 6801911819.374537, "median": 6799910912.0, "minimum": 5833928704, "sample_count": 14044 }, "gpu_board_watts": { "maximum": 305.13, "mean": 299.9847897001666, "median": 299.99, "minimum": 283.16, "sample_count": 14408 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 14408 }, "gpu_sm_clock_mhz": { "maximum": 1792.0, "mean": 1531.5502498611881, "median": 1545.0, "minimum": 1477.0, "sample_count": 14408 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.21002220988339, "median": 100.0, "minimum": 50.0, "sample_count": 14408 }, "host_mem_available_bytes": { "maximum": 96192446464, "mean": 95707652121.37396, "median": 95703994368.0, "minimum": 95185977344, "sample_count": 14044 }, "process_pss_bytes": { "maximum": 1000995840, "mean": 994586522.5041299, "median": 999561216.0, "minimum": 939293696, "sample_count": 14044 }, "process_rss_bytes": { "maximum": 1003978752, "mean": 997597475.3631444, "median": 1002545152.0, "minimum": 942276608, "sample_count": 14044 } }, "process_sample_count": 14044, "response_union_seconds": 14409.160561117635, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 14311424 }, "observed_endpoint_seconds": 13820.278715579421, "observed_intervals": 576, "requested_intervals": 576 } }, "response_outer_span_seconds": 14418.665552712977, "response_seconds": { "maximum": 25.164631637977436, "mean": 24.864257757205955, "median": 24.896307882503606, "minimum": 23.494174277991988, "p95": 25.100366784119977, "sample_count": 2304 }, "response_window_union_seconds": 14409.160561117635, "server_decode_tokens_per_second": { "maximum": 25.484744831743715, "mean": 18.02991749691153, "median": 17.848019291580243, "minimum": 11.97676620079551, "sample_count": 2304 }, "synthetic_first_token_seconds": { "maximum": 14.590205368993338, "mean": 9.797891346179211, "median": 10.10864435750409, "minimum": 3.4045858300523832, "p95": 14.525741860616836, "sample_count": 2304 }, "warmup_completed_rounds": 1, "warmup_requests_per_round": 4 } ], "id": "sustained-32b", "model": "qwen3-32b-q4", "module": "sustained", "placement": { "buffers": [ { "device": "CPU_Mapped", "reported_mib": 417.3 }, { "device": "CUDA0", "reported_mib": 18423.65 } ], "gpu_layers": 65, "meaning": "Runtime buffer allocation is not measured resident process RAM.", "requested_cpu_expert_layers": 0, "total_layers": 65 }, "record_sha256": "79abc8b59d526e04286b44ba1fdecc454ad81d540397d9dad2c2634dda632816", "resources_by_phase": { "baseline": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 2, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 5514604544, "mean": 5505738752, "median": 5505738752.0, "minimum": 5496872960, "sample_count": 2 }, "gpu_board_watts": { "maximum": 15.58, "mean": 15.555, "median": 15.555, "minimum": 15.53, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 180.0, "mean": 180.0, "median": 180.0, "minimum": 180.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96590258176, "mean": 96524697600, "median": 96524697600.0, "minimum": 96459137024, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1006686040200293, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "cleanup": { "gpu_energy": { "available": false, "joules": null, "reason": "unbracketed_boundary" }, "gpu_sample_count": 1, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 7321759744, "mean": 7300308992, "median": 7300308992.0, "minimum": 7278858240, "sample_count": 2 }, "gpu_board_watts": { "maximum": 108.08, "mean": 108.08, "median": 108.08, "minimum": 108.08, "sample_count": 1 }, "gpu_memory_used_bytes": { "maximum": 2097152.0, "mean": 2097152.0, "median": 2097152.0, "minimum": 2097152.0, "sample_count": 1 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2340.0, "median": 2340.0, "minimum": 2340.0, "sample_count": 1 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 1 }, "host_mem_available_bytes": { "maximum": 95783763968, "mean": 95729772544, "median": 95729772544.0, "minimum": 95675781120, "sample_count": 2 }, "process_pss_bytes": null, "process_rss_bytes": null }, "process_sample_count": 0, "response_union_seconds": 1.1560256039956585, "sampled_counter_deltas": { "deltas": null, "observed_endpoint_seconds": 0.0, "observed_intervals": 0, "requested_intervals": 1 } }, "load": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 213.13813387022867, "reason": null, "response_union_seconds": 3.6325854180031456, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 58.6739496376088, "trapezoid_pieces": 4, "watt_hours": 0.05920503718617463 }, "gpu_sample_count": 3, "host_sample_count": 2, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 5706874880, "mean": 5706758144, "median": 5706758144.0, "minimum": 5706641408, "sample_count": 2 }, "gpu_board_watts": { "maximum": 77.09, "mean": 62.513333333333335, "median": 76.77, "minimum": 33.68, "sample_count": 3 }, "gpu_memory_used_bytes": { "maximum": 19906166784.0, "mean": 19906166784.0, "median": 19906166784.0, "minimum": 19906166784.0, "sample_count": 3 }, "gpu_sm_clock_mhz": { "maximum": 2302.0, "mean": 2292.0, "median": 2302.0, "minimum": 2272.0, "sample_count": 3 }, "gpu_utilization_percent": { "maximum": 82.0, "mean": 79.33333333333333, "median": 82.0, "minimum": 74.0, "sample_count": 3 }, "host_mem_available_bytes": { "maximum": 96227672064, "mean": 96217997312, "median": 96217997312.0, "minimum": 96208322560, "sample_count": 2 }, "process_pss_bytes": { "maximum": 20099197952, "mean": 20099126272, "median": 20099126272.0, "minimum": 20099054592, "sample_count": 2 }, "process_rss_bytes": { "maximum": 20102180864, "mean": 20102109184, "median": 20102109184.0, "minimum": 20102037504, "sample_count": 2 } }, "process_sample_count": 2, "response_union_seconds": 3.6325854180031456, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 5, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 1.3338036570348777, "observed_intervals": 1, "requested_intervals": 1 } }, "ready": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 327.03305081688416, "reason": null, "response_union_seconds": 2.223970894992817, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 147.0491594800931, "trapezoid_pieces": 4, "watt_hours": 0.09084251411580116 }, "gpu_sample_count": 2, "host_sample_count": 3, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 7702061056, "mean": 6413938688, "median": 5770027008, "minimum": 5769728000, "sample_count": 3 }, "gpu_board_watts": { "maximum": 183.77, "mean": 136.445, "median": 136.445, "minimum": 89.12, "sample_count": 2 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29336010752.0, "median": 29336010752.0, "minimum": 29322379264.0, "sample_count": 2 }, "gpu_sm_clock_mhz": { "maximum": 2340.0, "mean": 2321.0, "median": 2321.0, "minimum": 2302.0, "sample_count": 2 }, "gpu_utilization_percent": { "maximum": 0.0, "mean": 0.0, "median": 0.0, "minimum": 0.0, "sample_count": 2 }, "host_mem_available_bytes": { "maximum": 96134074368, "mean": 95850448213.33333, "median": 96131477504, "minimum": 95285792768, "sample_count": 3 }, "process_pss_bytes": { "maximum": 1000994816, "mean": 919594325.3333334, "median": 878894080, "minimum": 878894080, "sample_count": 3 }, "process_rss_bytes": { "maximum": 1003978752, "mean": 922577578.6666666, "median": 881876992, "minimum": 881876992, "sample_count": 3 } }, "process_sample_count": 3, "response_union_seconds": 2.223970894992817, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 0 }, "observed_endpoint_seconds": 1.021025077032391, "observed_intervals": 1, "requested_intervals": 2 } }, "responses": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 4325377.306417703, "reason": null, "response_union_seconds": 14418.681973422004, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 299.98423672778705, "trapezoid_pieces": 14420, "watt_hours": 1201.4936962271397 }, "gpu_sample_count": 14419, "host_sample_count": 14050, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 7715704832, "mean": 6801967864.01936, "median": 6799910912.0, "minimum": 5833928704, "sample_count": 14050 }, "gpu_board_watts": { "maximum": 305.13, "mean": 299.9844940703239, "median": 299.99, "minimum": 283.16, "sample_count": 14419 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29349642240.0, "median": 29349642240.0, "minimum": 29349642240.0, "sample_count": 14419 }, "gpu_sm_clock_mhz": { "maximum": 1792.0, "mean": 1531.5317289687218, "median": 1545.0, "minimum": 1477.0, "sample_count": 14419 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 99.20694916429711, "median": 100.0, "minimum": 50.0, "sample_count": 14419 }, "host_mem_available_bytes": { "maximum": 96192446464, "mean": 95707636376.68896, "median": 95703994368.0, "minimum": 95185977344, "sample_count": 14050 }, "process_pss_bytes": { "maximum": 1000995840, "mean": 994587084.7635587, "median": 999561216.0, "minimum": 939293696, "sample_count": 14050 }, "process_rss_bytes": { "maximum": 1003978752, "mean": 997598025.9393594, "median": 1002545152.0, "minimum": 942276608, "sample_count": 14050 } }, "process_sample_count": 14050, "response_union_seconds": 14418.681973422004, "sampled_counter_deltas": { "deltas": { "major_faults": 0, "read_syscall_bytes": 0, "storage_read_bytes": 0, "storage_write_bytes": 17612800 }, "observed_endpoint_seconds": 14416.98345157504, "observed_intervals": 1, "requested_intervals": 1 } }, "warmup": { "gpu_energy": { "available": true, "interpolation": "linear_at_bracketed_boundaries", "joules": 6987.65928224834, "reason": null, "response_union_seconds": 23.74817975901533, "scope": "GPU board only; not whole-server AC energy", "time_weighted_mean_watts": 294.23978398157743, "trapezoid_pieces": 25, "watt_hours": 1.9410164672912056 }, "gpu_sample_count": 24, "host_sample_count": 23, "meaning": "Samples inside declared interval union. Means are arithmetic sample means; peaks need not coincide. Counter deltas exclude unsampled edges; not minimum required RAM.", "metrics": { "cgroup_memory_current_bytes": { "maximum": 5842784256, "mean": 5822150210.782609, "median": 5826940928, "minimum": 5801627648, "sample_count": 23 }, "gpu_board_watts": { "maximum": 306.76, "mean": 291.29333333333335, "median": 300.005, "minimum": 85.69, "sample_count": 24 }, "gpu_memory_used_bytes": { "maximum": 29349642240.0, "mean": 29348943189.333332, "median": 29349642240.0, "minimum": 29347545088.0, "sample_count": 24 }, "gpu_sm_clock_mhz": { "maximum": 2302.0, "mean": 1641.0416666666667, "median": 1635.0, "minimum": 1552.0, "sample_count": 24 }, "gpu_utilization_percent": { "maximum": 100.0, "mean": 98.875, "median": 99.0, "minimum": 90.0, "sample_count": 24 }, "host_mem_available_bytes": { "maximum": 96146788352, "mean": 96126758912, "median": 96126504960, "minimum": 96112836608, "sample_count": 23 }, "process_pss_bytes": { "maximum": 936446976, "mean": 929384670.6086956, "median": 933833728, "minimum": 910089216, "sample_count": 23 }, "process_rss_bytes": { "maximum": 939429888, "mean": 932429289.7391304, "median": 936816640, "minimum": 913072128, "sample_count": 23 } }, "process_sample_count": 23, "response_union_seconds": 23.74817975901533, "sampled_counter_deltas": { "deltas": { "major_faults": 8, "read_syscall_bytes": 0, "storage_read_bytes": 1441792, "storage_write_bytes": 20480 }, "observed_endpoint_seconds": 22.51987132197246, "observed_intervals": 1, "requested_intervals": 1 } } }, "reviewed_export_sha256": "7e9bf3881a4bd446b234cda9c158b651eb9d123d6af6090c39eed1e26ad459ac", "settings": { "batch": 2048, "cache_prompt": false, "cache_ram_bytes": 0, "concurrency": 4, "context_shift": false, "contexts": [ 8192 ], "endpoint": "/completion", "fit": "off", "flash_attention": true, "ignore_eos": true, "kv_cache": "f16", "lazy_mode": "off", "load_mode": "mmap", "measured_rounds": null, "mlock": false, "network_scope": "loopback", "output_tokens": 256, "p95_minimum_requests": 200, "placement": { "cpu_expert_layers": 0, "gpu_layers": "all" }, "reasoning": "not_applicable_raw_completion", "seed": 20260927, "server_slots": 4, "synthetic_prompt_sha256": "c099b5abc034050fac1133e0214990d1f72a124b64f36e8a80ba3209bc07d187", "temperature": 0, "threads": 16, "tokenization": "model_tokenizer_add_special_false_repeat_then_truncate", "total_context_tokens": 34816, "ubatch": 512, "warmup_rounds": 1 }, "status": "completed", "telemetry": { "gpu_samples": 14451, "host_samples": 14082, "maximum_gpu_gap_seconds": 1.0334704880369827, "maximum_host_gap_seconds": 1.5620193640352227, "meaning": "Sampled observations; no numeric temperatures published and no warranty between samples.", "thermal_slowdown_fully_available": true, "thermal_slowdown_observed_samples": 0, "thermal_slowdown_states": { "hardware": { "active": 0, "not_active": 14451, "unavailable": 0 }, "software": { "active": 0, "not_active": 14451, "unavailable": 0 } } } } ], "comparisons": [ { "axis": "concurrency", "cases": [ "concurrency-8b-c1", "concurrency-8b-c4" ] }, { "axis": "concurrency", "cases": [ "concurrency-32b-c1", "concurrency-32b-c4" ] }, { "axis": "concurrency", "cases": [ "concurrency-72b-q4-c1", "concurrency-72b-q4-c4" ] } ], "environment": { "kernel": "6.12.0-211.56.1.el10_2.0.1.x86_64", "nvidia_driver": "595.91.07", "os": { "id": "rocky", "name": "Rocky Linux", "version_id": "10.2" } }, "hardware": { "cpu": "AMD EPYC 4565P 16-Core Processor", "fan_profile": "GPU300WAuto", "gpu": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", "gpu_nominal_vram_gb": 96, "gpu_vram_bytes": 102641958912, "installed_ram_bytes": 103079215104, "kernel_mem_total_bytes": 100271353856, "logical_cpu_count": 32, "physical_cpu_count": 16 }, "historical_results_pooled": false, "limitations": [ "Externally pinned capture integrity is not remote hardware attestation.", "Five repeated rounds share a process and are not five independent installations.", "Synthetic token latency is not practical first-visible answer latency.", "GPU board power is not whole-server AC power; energy is unavailable without bracketing samples.", "Sampled memory peaks may not coincide and do not establish minimum required system RAM.", "The practical corpus contains 24 authored tasks, not a broad model-quality benchmark.", "Engine telemetry includes process loading and internal warmup; exact per-sample resource windows are unavailable.", "No historical result is relabeled or pooled into this versioned series." ], "models": [ { "architecture": "dense", "chat_template_sha256": "57f1fd00f0013a2be96aa79b857391f27e23df5b5f847072b524c897e24d0361", "effective_chat_template_rule": "llama.cpp-7fe450e19-chat-source-v1", "effective_chat_template_sha256": "57f1fd00f0013a2be96aa79b857391f27e23df5b5f847072b524c897e24d0361", "files": [ { "bytes": 5027783488, "sha256": "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785" } ], "id": "qwen3-8b-q4", "name": "Qwen3-8B", "quantization": "Q4_K_M", "repository": "Qwen/Qwen3-8B-GGUF", "revision": "7c41481f57cb95916b40956ab2f0b139b296d974", "tokenizer_sha256": "2629971f294a1a43ef545d689a2090a820e0e1a15e5371e1c88d1c21159ef849" }, { "architecture": "dense", "chat_template_sha256": "57f1fd00f0013a2be96aa79b857391f27e23df5b5f847072b524c897e24d0361", "effective_chat_template_rule": "llama.cpp-7fe450e19-chat-source-v1", "effective_chat_template_sha256": "57f1fd00f0013a2be96aa79b857391f27e23df5b5f847072b524c897e24d0361", "files": [ { "bytes": 19762149024, "sha256": "efd971561896866f0e910cce52761ca77b1b138090c7f15fe284676d57d1f689" } ], "id": "qwen3-32b-q4", "name": "Qwen3-32B", "quantization": "Q4_K_M", "repository": "Qwen/Qwen3-32B-GGUF", "revision": "938a7432affaec9157f883a87164e2646ae17555", "tokenizer_sha256": "2629971f294a1a43ef545d689a2090a820e0e1a15e5371e1c88d1c21159ef849" }, { "architecture": "dense", "chat_template_sha256": "d5495a1e5db0611132a97e46a65dbb64a642a499421228b9c8b93229097fa9a4", "effective_chat_template_rule": "llama.cpp-7fe450e19-chat-source-v1", "effective_chat_template_sha256": "4e9918361c284a93880606d182d64da6a9fe97cdc1f5c5a78c1c8840444246fc", "files": [ { "bytes": 3986869120, "sha256": "3531a90691b5f4edb4bdf488246d72ce0b5d40cacfab1489718a662cec386e43" }, { "bytes": 3877864544, "sha256": "a4cfb079461591e2c23ef73953aeba4b2520a9b93afaa841f31adf4e480647ab" }, { "bytes": 3963622720, "sha256": "c67c3641d018b7ca646e7c37b50152687d4b52ce6ab6bf170b006a756b52778d" }, { "bytes": 3941463040, "sha256": "92f5149d685ad5c5ab0fc30f5e0725d95a746d9a8e54d8397707b97dc435cd76" }, { "bytes": 3963688384, "sha256": "f5abb6ecd7f3ec4438a4e89ae704acd43405182a0477776d5e8aa35a71a5ecdc" }, { "bytes": 3941463040, "sha256": "ead67a4ac2648b0f7b71fea755f910ed208f3f8b881b55c208fc3fdca67359ee" }, { "bytes": 3826849152, "sha256": "f7b168a5fdf78695736a6441dc846b19251339750ea78bf993057b7bf499671d" }, { "bytes": 3983901792, "sha256": "4ab253ada85c14097597546ac13cfb1426cb175d1d8da520b7a2904d0c323e32" }, { "bytes": 3921183968, "sha256": "51661b80af773b488e807a03fc7ea31bb64630c50e3ee77fbb5e37bdd9ff9979" }, { "bytes": 3995005888, "sha256": "fcc180433606a81e2137ba1f088defe3ad51ec9353f5d129a7209614597e16f9" }, { "bytes": 3586683584, "sha256": "57b9437649f12062bdbed6c623c3f6adc6924a3d2f4e7aabdd3ab32883f5485d" }, { "bytes": 1021870240, "sha256": "60ae16428f9edb62b60e6c8657ba9344c6fae6ef44c497845dc6a6fd002e4d5b" } ], "id": "qwen25-72b-q4", "name": "Qwen2.5-72B-Instruct", "quantization": "Q4_K_M", "repository": "Qwen/Qwen2.5-72B-Instruct-GGUF", "revision": "7ca3bc388f97b264c4283bc9bf1055e2abc38441", "tokenizer_sha256": "67b3d77a05acfb80f96742cec2740cc6ee8fffadef98e655e855baf58a3946b1" } ], "protocol": "gsh-llm-core/1.1", "runtime": { "archives_sha256": [ "c2ab9e19838513ff69d1af8d999ad717dd3c7ee4714ac04c7ed5ab9077c50e4e", "1466daea60aad1144819e151b2bae19d54556cf1da6c129c4f55a5ded2637c25" ], "binary_sha256": "376626f953efd7bbf16dee112747196d2b1de26717bcd1d37e203340dc5c043c", "commit": "7fe450e19305b828c199d602c23a8337aaa1f03b", "name": "llama.cpp", "release": "b11146" }, "schema": "gsh-llm-core-article/1", "sources": [ { "exporter_sha256": "4f4046b91d99ba35175c3c6c04473a0021f54c6baa4892625cb66435f4c7e2ec", "finished_at": "2026-09-27T15:11:44.063808+00:00", "plan_id": "core-core-20260927t135400-core", "plan_sha256": "d6b25477667ae18cdbc23760f8a5540291e3d88246734ade84d167a32fc9011c", "protocol_sha256": "3ce8ce9f2b17e66a655c07fd7e7aa3f1762790ad9be38f87d29e5e1279dec3ac", "record_sha256": "fb26d7e444a2a826e9f652b7e048338d0bb8ddb92c1672ae6b919fc2334d492c", "reviewed_export_sha256": "59f09a9b924c4cb4ae1370f85109347542e72457c07f0a7a8ff351b31fedab64", "runner_sha256": "1bac2436c5aa2eca4f4b42cc9b8f0942b5c66256eb1a41a2169b88c12b044f4e", "started_at": "2026-09-27T13:56:01.723397+00:00", "task_corpus_sha256": "0642db889565ce25b980e24afa5048eb83e4d15c36754457474a65b2c1f71c21" }, { "exporter_sha256": "4f4046b91d99ba35175c3c6c04473a0021f54c6baa4892625cb66435f4c7e2ec", "finished_at": "2026-09-28T00:41:20.654174+00:00", "plan_id": "core-sustained-20260927t203551-sustained79182232", "plan_sha256": "1f23d7c3b12fbcd78b46e277e64c415da613fa72c7fc9d831d17ae6859cf87be", "protocol_sha256": "3ce8ce9f2b17e66a655c07fd7e7aa3f1762790ad9be38f87d29e5e1279dec3ac", "record_sha256": "8941ad01c40078a4ab873ea0b80a24088e0a3fe1cd6bf76c20f728bd2f8122a0", "reviewed_export_sha256": "7e9bf3881a4bd446b234cda9c158b651eb9d123d6af6090c39eed1e26ad459ac", "runner_sha256": "1bac2436c5aa2eca4f4b42cc9b8f0942b5c66256eb1a41a2169b88c12b044f4e", "started_at": "2026-09-27T20:36:24.408847+00:00", "task_corpus_sha256": "0642db889565ce25b980e24afa5048eb83e4d15c36754457474a65b2c1f71c21" } ], "status": "completed", "test_date_timezone": "Europe/Warsaw", "tested_at": "2026-09-28" }