{
  "schema": 2,
  "kind": "thermal90_campaign",
  "publisher": "GPU Server Hub",
  "campaign": "short",
  "ok": false,
  "reason_code": "foreign_process",
  "record_sha256": "005a81b9500e6e28a39bf0597053f5c0adcd934ceaf75419a502eba7c91fb80d",
  "policy": {
    "id": "thermal90-margin2-v1",
    "temperature_abort_c": 90,
    "thermal_margin_abort_at_or_below_c": 2,
    "sample_interval_seconds": 1,
    "telemetry_timeout_seconds": 2,
    "power_limit_w": 300,
    "minimum_free_bytes": 26843545600,
    "minimum_available_ram_bytes": 4294967296,
    "maximum_sample_gap_seconds": 3,
    "auxiliary_monitor_timeout_seconds": 8
  },
  "started_at": "2026-09-26T13:01:43.041446+00:00",
  "finished_at": "2026-09-26T13:04:29.972340+00:00",
  "campaign_wall_seconds_including_setup_warmup_cleanup": 166.9311071980046,
  "requested_sustained_measured_seconds": 0,
  "runs": [
    {
      "run": "01-api-small",
      "kind": "api",
      "status": "stopped",
      "ok": false,
      "reason_code": "foreign_process",
      "record_sha256": "a30fdd35c176e653dfd587e5ee8048e7fa77ccabb89d4d04c1ac1149995235f9",
      "policy": {
        "id": "thermal90-margin2-v1",
        "temperature_abort_c": 90,
        "thermal_margin_abort_at_or_below_c": 2,
        "sample_interval_seconds": 1,
        "telemetry_timeout_seconds": 2,
        "power_limit_w": 300,
        "minimum_free_bytes": 26843545600,
        "minimum_available_ram_bytes": 4294967296,
        "maximum_sample_gap_seconds": 3,
        "auxiliary_monitor_timeout_seconds": 8
      },
      "model": {
        "name": "Qwen3-8B",
        "repository": "Qwen/Qwen3-8B-GGUF",
        "revision": "7c41481f57cb95916b40956ab2f0b139b296d974",
        "license": "apache-2.0",
        "quantization": "Q4_K_M",
        "expected_total_bytes": 5027783488,
        "expected_hashes": [
          "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785"
        ]
      },
      "tool": {
        "name": "llama.cpp",
        "version": "0.5.0-dev",
        "release": "b11146",
        "commit": "7fe450e19305b828c199d602c23a8337aaa1f03b"
      },
      "hardware": {
        "gpu": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition",
        "driver": "595.91.07",
        "kernel": "6.12.0-211.56.1.el10_2.0.1.x86_64",
        "cpu": [
          "AMD EPYC 4565P 16-Core Processor"
        ],
        "logical_cpus": 32,
        "linux_visible_ram_bytes": 100271353856,
        "physical_system_ram_gb": 96,
        "planned_system_ram_gb_not_measured": 192,
        "gpu_power_limit_w": 300,
        "gpu_reported_memory_mib": 97887.0
      },
      "settings": {
        "contexts": [
          2048,
          8192,
          32768
        ],
        "concurrency": [
          1,
          4
        ],
        "generated_tokens": 256,
        "repetitions": 3,
        "duration_seconds": 0,
        "prompt_cache": false,
        "temperature": 0,
        "seed_base": 20260926,
        "kv_cache": "f16",
        "network_scope": "loopback-only",
        "requested_gpu_layers": 999,
        "fit": "off",
        "flash_attention": "on",
        "threads": 16,
        "batch": 2048,
        "ubatch": 512,
        "server_slots": 4,
        "context_per_slot": 33280,
        "total_context_tokens": 133120,
        "cooldown_before_run_to_c": 40
      },
      "started_at": "2026-09-26T13:01:47.696121+00:00",
      "finished_at": "2026-09-26T13:04:29.963144+00:00",
      "campaign_wall_seconds_including_cooldown_warmup_cleanup": 162.24360672401963,
      "trigger_observations": [
        {
          "scope": "run",
          "reason_code": "foreign_process",
          "recorded_at": "2026-09-26T13:04:29.962477+00:00",
          "sample_available": true,
          "sample": {
            "temperature_c": 85.0,
            "thermal_margin_c": 7.0,
            "gpu_board_power_w": 300.05,
            "gpu_power_limit_w": 300.0,
            "gpu_memory_used_mib": 23962.0,
            "gpu_utilization_percent": 98.0,
            "sm_clock_mhz": 1747.0,
            "memory_clock_mhz": 13365.0,
            "fan_percent": 59.0,
            "software_thermal_slowdown": "not_active",
            "hardware_thermal_slowdown": "not_active",
            "software_power_cap": "active"
          },
          "sampled_at": "2026-09-26T13:04:29.783048+00:00",
          "interpretation": "Campaign stopped for the recorded harness reason; not a general hardware diagnosis."
        }
      ],
      "telemetry_intervals": [
        {
          "scope": "run",
          "meaning": "Sampled monitor interval includes loading, warmup and cleanup; not only completed measured requests.",
          "sample_count": 163,
          "nominal_interval_seconds": 1,
          "metrics": {
            "temperature_c": {
              "sample_count": 163,
              "minimum": 29.0,
              "maximum": 87.0,
              "mean": 70.66257668711657
            },
            "thermal_margin_c": {
              "sample_count": 163,
              "minimum": 5.0,
              "maximum": 63.0,
              "mean": 21.34355828220859
            },
            "gpu_board_power_w": {
              "sample_count": 163,
              "minimum": 14.32,
              "maximum": 302.21,
              "mean": 296.20184049079757
            },
            "gpu_power_limit_w": {
              "sample_count": 163,
              "minimum": 300.0,
              "maximum": 300.0,
              "mean": 300.0
            },
            "gpu_memory_used_mib": {
              "sample_count": 163,
              "minimum": 2.0,
              "maximum": 23962.0,
              "mean": 23698.39263803681
            },
            "gpu_utilization_percent": {
              "sample_count": 163,
              "minimum": 0.0,
              "maximum": 100.0,
              "mean": 97.60122699386503
            },
            "sm_clock_mhz": {
              "sample_count": 163,
              "minimum": 180.0,
              "maximum": 2295.0,
              "mean": 1665.4049079754602
            },
            "memory_clock_mhz": {
              "sample_count": 163,
              "minimum": 405.0,
              "maximum": 13365.0,
              "mean": 13285.490797546012
            },
            "fan_percent": {
              "sample_count": 163,
              "minimum": 30.0,
              "maximum": 59.0,
              "mean": 43.63803680981595
            }
          },
          "states": {
            "software_thermal_slowdown": {
              "active": 0,
              "not_active": 163,
              "unavailable": 0
            },
            "hardware_thermal_slowdown": {
              "active": 0,
              "not_active": 163,
              "unavailable": 0
            },
            "software_power_cap": {
              "active": 161,
              "not_active": 2,
              "unavailable": 0
            }
          },
          "first_sample_at": "2026-09-26T13:01:47.784291+00:00",
          "last_sample_at": "2026-09-26T13:04:29.783048+00:00",
          "maximum_observed_gap_seconds": 1.0066309139947407,
          "observed_inter_sample_gaps_seconds": {
            "sample_count": 162,
            "minimum": 0.9841931780101731,
            "maximum": 1.0066309139947407,
            "mean": 0.9999923081296054
          }
        }
      ],
      "groups": [
        {
          "context": 2048,
          "concurrency": 1,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 3,
          "sum_completed_round_seconds": 4.4588610870123375,
          "first_completed_round_started_at": "2026-09-26T13:01:50.524567+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:01:54.984634+00:00",
          "completed_rounds_timestamp_span_seconds": 4.460067,
          "performance": {
            "aggregate_tokens_per_second_mean": 172.2413486732458,
            "ttft_median_seconds": 0.19229250799980946,
            "response_median_seconds": 1.4858914249925874
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:01:50.524567+00:00",
            "finished_at": "2026-09-26T13:01:54.984634+00:00",
            "elapsed_seconds": 4.460066694009583,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        },
        {
          "context": 2048,
          "concurrency": 4,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 12,
          "sum_completed_round_seconds": 9.741589892015327,
          "first_completed_round_started_at": "2026-09-26T13:01:56.475102+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:02:06.217920+00:00",
          "completed_rounds_timestamp_span_seconds": 9.742818,
          "performance": {
            "aggregate_tokens_per_second_mean": 315.35226148376654,
            "ttft_median_seconds": 0.6916851725109154,
            "response_median_seconds": 3.2339182024879847
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:01:56.475102+00:00",
            "finished_at": "2026-09-26T13:02:06.217920+00:00",
            "elapsed_seconds": 9.742815776000498,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        },
        {
          "context": 8192,
          "concurrency": 1,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 3,
          "sum_completed_round_seconds": 7.060860039026011,
          "first_completed_round_started_at": "2026-09-26T13:02:08.564545+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:02:15.626780+00:00",
          "completed_rounds_timestamp_span_seconds": 7.062235,
          "performance": {
            "aggregate_tokens_per_second_mean": 108.76876119157484,
            "ttft_median_seconds": 0.8387344900111202,
            "response_median_seconds": 2.3523099700105377
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:02:08.564545+00:00",
            "finished_at": "2026-09-26T13:02:15.626780+00:00",
            "elapsed_seconds": 7.062233739008661,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        },
        {
          "context": 8192,
          "concurrency": 4,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 12,
          "sum_completed_round_seconds": 19.641528482025024,
          "first_completed_round_started_at": "2026-09-26T13:02:17.986216+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:02:37.629244+00:00",
          "completed_rounds_timestamp_span_seconds": 19.643028,
          "performance": {
            "aggregate_tokens_per_second_mean": 156.4053134595578,
            "ttft_median_seconds": 2.462668563995976,
            "response_median_seconds": 6.507020510485745
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:02:17.986216+00:00",
            "finished_at": "2026-09-26T13:02:37.629244+00:00",
            "elapsed_seconds": 19.643023708020337,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        },
        {
          "context": 32768,
          "concurrency": 1,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 3,
          "sum_completed_round_seconds": 21.359202271036338,
          "first_completed_round_started_at": "2026-09-26T13:02:44.662428+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:03:06.022909+00:00",
          "completed_rounds_timestamp_span_seconds": 21.360481,
          "performance": {
            "aggregate_tokens_per_second_mean": 35.95710118073934,
            "ttft_median_seconds": 4.7556803959887475,
            "response_median_seconds": 7.121359912009211
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:02:44.662428+00:00",
            "finished_at": "2026-09-26T13:03:06.022909+00:00",
            "elapsed_seconds": 21.360483740019845,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        },
        {
          "context": 32768,
          "concurrency": 4,
          "status": "completed",
          "requested_duration_seconds": 0,
          "completed_rounds": 3,
          "completed_responses": 12,
          "sum_completed_round_seconds": 76.73059668100905,
          "first_completed_round_started_at": "2026-09-26T13:03:13.206040+00:00",
          "last_completed_round_finished_at": "2026-09-26T13:04:29.939088+00:00",
          "completed_rounds_timestamp_span_seconds": 76.733048,
          "performance": {
            "aggregate_tokens_per_second_mean": 40.03793431846682,
            "ttft_median_seconds": 12.861717290492379,
            "response_median_seconds": 25.30321582951001
          },
          "measured_case_window": {
            "started_at": "2026-09-26T13:03:13.206040+00:00",
            "finished_at": "2026-09-26T13:04:29.939088+00:00",
            "elapsed_seconds": 76.73304581898265,
            "meaning": "Case measurement loop after its warmup, not time continuously generating tokens."
          }
        }
      ],
      "completed_rounds": 18,
      "completed_responses": 45,
      "sum_completed_round_seconds": 138.99263845212408,
      "first_completed_round_started_at": "2026-09-26T13:01:50.524567+00:00",
      "last_completed_round_finished_at": "2026-09-26T13:04:29.939088+00:00",
      "completed_rounds_timestamp_span_seconds": 159.414521,
      "completed_case_groups": 6,
      "maximum_inter_round_gap_seconds": 7.183131,
      "sum_inter_round_gap_seconds": 20.421842,
      "reported_measured_outer_window_seconds": 159.41452070800005,
      "measured_outer_window_meaning": "First case measurement start through last completed round; can span later case warmups and intervals. Not a sum of active generation time.",
      "measured_started_at": "2026-09-26T13:01:50.524567+00:00",
      "measured_finished_at": "2026-09-26T13:04:29.939088+00:00"
    }
  ],
  "plan_status": [
    {
      "run": "01-api-small",
      "kind": "api",
      "model": "Qwen3-8B",
      "status": "stopped"
    },
    {
      "run": "02-api-medium",
      "kind": "api",
      "model": "Qwen3-32B",
      "status": "not_run",
      "meaning": "No completed model test is claimed; this is not a failed result for this model."
    },
    {
      "run": "03-engine-large",
      "kind": "engine",
      "model": "Qwen2.5-72B-Instruct",
      "status": "not_run",
      "meaning": "No completed model test is claimed; this is not a failed result for this model."
    },
    {
      "run": "04-api-large",
      "kind": "api",
      "model": "Qwen2.5-72B-Instruct",
      "status": "not_run",
      "meaning": "No completed model test is claimed; this is not a failed result for this model."
    }
  ],
  "limitations": [
    "Only records from this SHA-verified thermal90 policy campaign are included; no earlier 85 C results or rounds are merged.",
    "A harness stop is not itself proof of GPU failure, thermal shutdown, or the absence of hardware faults.",
    "Requested duration, measured request window, and campaign duration including warmup/cleanup are distinct.",
    "One-second telemetry is sampled and may miss events between samples. GPU board power is not whole-server AC power.",
    "Skipped or pending models were not tested in this campaign; they are not failed model tests.",
    "Physical system RAM during these measurements is 96 GB; the offered 192 GB configuration was not measured."
  ]
}