[ { "schema_version": 1, "generated_at": "2026-08-03T12:15:51.105587+00:00", "build": { "amd_aorta_version": "0.2.2rc20260803", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "30812111757", "head_sha": "89176cdfac22278abc159e94c47aed7c1d663255", "wheel_file": "amd_aorta-0.2.2rc20260803-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 254.41335421055555, "mean_wall_clock_sec": 5.2149085234850645, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-03T12-06-48/matrix.json", "duration_sec": 16.38935, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 39.70108600333333, "mean_wall_clock_sec": 11.69123553019017, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.127894502133131, "decode_latency_ms": 1.01258909149516, "tokens_per_sec": 3950.269693399239, "decoded_tokens": 496.0, "step_time_p50": 37.057443987578154, "step_time_p99": 47.82466470263898, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-03T12-07-04/matrix.json", "duration_sec": 23.195656, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1516.520546283573, "mean_wall_clock_sec": 21.413878113031387, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.4421623200178146, "step_time_p99": 44.06274572014809 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-03T12-07-28/matrix.json", "duration_sec": 35.266733, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1422.592904418707, "mean_wall_clock_sec": 17.09160147793591, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.475670263171196, "step_time_p99": 29.19031471014023 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-03T12-08-04/matrix.json", "duration_sec": 30.811549, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2229.8173427581787, "mean_wall_clock_sec": 18.145053784362972, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2229.8173427581787, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-03T12-08-35/matrix.json", "duration_sec": 31.35832, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11399.29648861289, "mean_wall_clock_sec": 23.018114496953785, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-03T12-09-06/matrix.json", "duration_sec": 105.760543, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11239.49496075511, "mean_wall_clock_sec": 21.111012200824916, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-03T12-09-06/matrix.json", "duration_sec": 105.760543, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11606.56196065247, "mean_wall_clock_sec": 23.02234795689583, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-03T12-09-06/matrix.json", "duration_sec": 105.760543, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12661.723216064274, "mean_wall_clock_sec": 25.67394043970853, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-03T12-09-06/matrix.json", "duration_sec": 105.760543, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1539.2195945605636, "mean_wall_clock_sec": 25.03456206060946, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.424071379005909, "step_time_p99": 43.29227281734347 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-03T12-10-52/matrix.json", "duration_sec": 45.236129, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1935.6966144405305, "mean_wall_clock_sec": 25.52133037801832, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.954441778361797, "step_time_p99": 9.382011760026217 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-03T12-11-37/matrix.json", "duration_sec": 45.476026, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 3879.2184352874756, "mean_wall_clock_sec": 31.822418483905494, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3879.2184352874756, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-03T12-12-23/matrix.json", "duration_sec": 45.636593, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18809.17681567371, "mean_wall_clock_sec": 37.68191348016262, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-03T12-13-08/matrix.json", "duration_sec": 163.527905, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18528.86881865561, "mean_wall_clock_sec": 35.48480960633606, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-03T12-13-08/matrix.json", "duration_sec": 163.527905, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18698.581118136644, "mean_wall_clock_sec": 37.3422136195004, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-03T12-13-08/matrix.json", "duration_sec": 163.527905, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 19125.48066303134, "mean_wall_clock_sec": 39.760336339473724, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-03T12-13-08/matrix.json", "duration_sec": 163.527905, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-04T11:54:07.241201+00:00", "build": { "amd_aorta_version": "0.2.2rc20260804", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "30906042875", "head_sha": "5589d14dd674d91ea3792988cfaa94e280ae2957", "wheel_file": "amd_aorta-0.2.2rc20260804-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 261.36021316051483, "mean_wall_clock_sec": 5.211701674386859, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-04T11-45-04/matrix.json", "duration_sec": 16.38583, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 39.401941234245896, "mean_wall_clock_sec": 11.624807843007147, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.134257996454835, "decode_latency_ms": 1.0084007235784684, "tokens_per_sec": 3966.677042639727, "decoded_tokens": 496.0, "step_time_p50": 37.34056884422898, "step_time_p99": 47.30467809364199, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-04T11-45-21/matrix.json", "duration_sec": 23.198077, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1496.8912121839821, "mean_wall_clock_sec": 21.223704744130373, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3126851990818977, "step_time_p99": 43.62073954194784 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-04T11-45-45/matrix.json", "duration_sec": 35.010607, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1440.4280502349138, "mean_wall_clock_sec": 17.14773778617382, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 5.6952377781271935, "step_time_p99": 7.992206420749426 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-04T11-46-20/matrix.json", "duration_sec": 31.153647, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2268.842029571533, "mean_wall_clock_sec": 18.63477344531566, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2268.842029571533, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-04T11-46-51/matrix.json", "duration_sec": 31.203311, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11699.665186926723, "mean_wall_clock_sec": 23.08767331764102, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-04T11-47-22/matrix.json", "duration_sec": 105.961976, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11434.67472679913, "mean_wall_clock_sec": 21.224832001142204, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-04T11-47-22/matrix.json", "duration_sec": 105.961976, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 11703.825123608112, "mean_wall_clock_sec": 23.19548445381224, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-04T11-47-22/matrix.json", "duration_sec": 105.961976, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12475.967891514301, "mean_wall_clock_sec": 25.63562870863825, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-04T11-47-22/matrix.json", "duration_sec": 105.961976, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1515.3193115256727, "mean_wall_clock_sec": 25.07277488987893, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.6094185411930084, "step_time_p99": 53.141798712313175 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-04T11-49-08/matrix.json", "duration_sec": 44.787035, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1937.4379392247647, "mean_wall_clock_sec": 25.780079113319516, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 7.09398090839386, "step_time_p99": 38.59115332365036 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-04T11-49-53/matrix.json", "duration_sec": 45.48511, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4007.4175834655766, "mean_wall_clock_sec": 32.518171062693, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4007.417583465576, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-04T11-50-38/matrix.json", "duration_sec": 45.687005, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18518.58223695308, "mean_wall_clock_sec": 37.5170262074098, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-04T11-51-24/matrix.json", "duration_sec": 163.661649, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18225.139918737113, "mean_wall_clock_sec": 35.44473580084741, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-04T11-51-24/matrix.json", "duration_sec": 163.661649, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 18461.202145554125, "mean_wall_clock_sec": 37.21057320293039, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-04T11-51-24/matrix.json", "duration_sec": 163.661649, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 19801.022356376052, "mean_wall_clock_sec": 39.783909467980266, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-04T11-51-24/matrix.json", "duration_sec": 163.661649, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-05T11:55:03.847946+00:00", "build": { "amd_aorta_version": "0.2.2rc20260805", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31002522044", "head_sha": "a417572547ca7f62ce0d66ca62bbfd88b24bef62", "wheel_file": "amd_aorta-0.2.2rc20260805-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 264.89886175841093, "mean_wall_clock_sec": 5.2589089861139655, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-05T11-45-39/matrix.json", "duration_sec": 16.439026, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 40.89358286000788, "mean_wall_clock_sec": 12.428017090074718, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1447675060480833, "decode_latency_ms": 1.0446482488224584, "tokens_per_sec": 3829.04006636574, "decoded_tokens": 496.0, "step_time_p50": 38.32078771665692, "step_time_p99": 49.016060074791305, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-05T11-45-55/matrix.json", "duration_sec": 24.051124, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1674.955191090703, "mean_wall_clock_sec": 23.0530548337847, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.6306079924106598, "step_time_p99": 3.0193189531564713 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-05T11-46-20/matrix.json", "duration_sec": 37.069856, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1609.5772518310696, "mean_wall_clock_sec": 18.06601799838245, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.3014479130506516, "step_time_p99": 47.88430852815509 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-05T11-46-57/matrix.json", "duration_sec": 32.363185, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2423.7818717956543, "mean_wall_clock_sec": 19.47625693678856, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2423.7818717956543, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-05T11-47-30/matrix.json", "duration_sec": 32.168372, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12653.14285736531, "mean_wall_clock_sec": 24.246596546843648, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-05T11-48-02/matrix.json", "duration_sec": 111.131252, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12833.191732876003, "mean_wall_clock_sec": 22.594846926629543, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-05T11-48-02/matrix.json", "duration_sec": 111.131252, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12682.386813685298, "mean_wall_clock_sec": 24.681888478808105, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-05T11-48-02/matrix.json", "duration_sec": 111.131252, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13307.354992255569, "mean_wall_clock_sec": 26.91842984687537, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-05T11-48-02/matrix.json", "duration_sec": 111.131252, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1788.901953957975, "mean_wall_clock_sec": 27.112858799286187, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3335132971405983, "step_time_p99": 39.07612472772598 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-05T11-49-53/matrix.json", "duration_sec": 47.283202, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2155.5741000920534, "mean_wall_clock_sec": 27.449993884190917, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.9908518344163895, "step_time_p99": 34.46580255404115 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-05T11-50-40/matrix.json", "duration_sec": 47.288758, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4368.022108078003, "mean_wall_clock_sec": 33.93932637851685, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4368.022108078003, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-05T11-51-27/matrix.json", "duration_sec": 46.888553, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20173.54510165751, "mean_wall_clock_sec": 38.705810504965484, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-05T11-52-14/matrix.json", "duration_sec": 170.05368, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20373.132753185928, "mean_wall_clock_sec": 36.842141257598996, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-05T11-52-14/matrix.json", "duration_sec": 170.05368, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20910.31469963491, "mean_wall_clock_sec": 39.76071457564831, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-05T11-52-14/matrix.json", "duration_sec": 170.05368, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20970.836475491524, "mean_wall_clock_sec": 41.15475586708635, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-05T11-52-14/matrix.json", "duration_sec": 170.05368, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-06T11:56:40.797499+00:00", "build": { "amd_aorta_version": "0.2.2rc20260806", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31098580469", "head_sha": "0bf0f1d5760b86bb112d310636aadb60e53101ce", "wheel_file": "amd_aorta-0.2.2rc20260806-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 258.7148603051901, "mean_wall_clock_sec": 5.217439249157906, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-06T11-47-13/matrix.json", "duration_sec": 16.437277, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 39.63245660997927, "mean_wall_clock_sec": 12.357576319016516, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.126008341088891, "decode_latency_ms": 1.0104902208812776, "tokens_per_sec": 3958.4747257736794, "decoded_tokens": 496.0, "step_time_p50": 37.09761332720518, "step_time_p99": 47.445825962349765, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-06T11-47-29/matrix.json", "duration_sec": 23.953362, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1686.9661586824805, "mean_wall_clock_sec": 23.103123735636473, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 2.019866369664669, "step_time_p99": 3.5094156488776207 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-06T11-47-54/matrix.json", "duration_sec": 37.169879, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1598.4002414625138, "mean_wall_clock_sec": 18.082872598432004, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.778175942599773, "step_time_p99": 47.267242427915335 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-06T11-48-32/matrix.json", "duration_sec": 32.364817, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2505.229139328003, "mean_wall_clock_sec": 19.773910203948617, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2505.229139328003, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-06T11-49-04/matrix.json", "duration_sec": 32.513845, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12358.65183826536, "mean_wall_clock_sec": 23.913750540465117, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-06T11-49-36/matrix.json", "duration_sec": 111.837699, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12806.768736802042, "mean_wall_clock_sec": 22.514728189446032, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-06T11-49-36/matrix.json", "duration_sec": 111.837699, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13103.491236455739, "mean_wall_clock_sec": 24.732631749473512, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-06T11-49-36/matrix.json", "duration_sec": 111.837699, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 14284.85409077257, "mean_wall_clock_sec": 28.115056235343218, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-06T11-49-36/matrix.json", "duration_sec": 111.837699, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1776.2116098310798, "mean_wall_clock_sec": 27.05523745343089, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.316002570092678, "step_time_p99": 41.59214800223708 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-06T11-51-28/matrix.json", "duration_sec": 47.085759, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2162.074110470712, "mean_wall_clock_sec": 27.37489387858659, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.8159447982907295, "step_time_p99": 56.65413187816739 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-06T11-52-16/matrix.json", "duration_sec": 47.34468, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4187.380027770996, "mean_wall_clock_sec": 33.44964222051203, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4187.380027770996, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-06T11-53-03/matrix.json", "duration_sec": 47.282505, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20145.061461254954, "mean_wall_clock_sec": 38.200943829491735, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-06T11-53-50/matrix.json", "duration_sec": 171.338784, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 21148.359899409115, "mean_wall_clock_sec": 37.49416956398636, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-06T11-53-50/matrix.json", "duration_sec": 171.338784, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 21153.759192675352, "mean_wall_clock_sec": 40.066207461990416, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-06T11-53-50/matrix.json", "duration_sec": 171.338784, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 21643.532000482082, "mean_wall_clock_sec": 41.765524642542005, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-06T11-53-50/matrix.json", "duration_sec": 171.338784, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-07T11:29:50.482146+00:00", "build": { "amd_aorta_version": "0.2.2rc20260807", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31173508703", "head_sha": "440dc27b6c4df9d77bdaa22774719d7649adf0bf", "wheel_file": "amd_aorta-0.2.2rc20260807-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 264.0826739370823, "mean_wall_clock_sec": 5.229185933247209, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-07T11-20-26/matrix.json", "duration_sec": 16.488591, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 36.97257488965988, "mean_wall_clock_sec": 12.098940684460104, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.092994585633278, "decode_latency_ms": 0.9475807929711957, "tokens_per_sec": 4221.2759372821, "decoded_tokens": 496.0, "step_time_p50": 34.273250959813595, "step_time_p99": 45.275920871645205, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-07T11-20-43/matrix.json", "duration_sec": 23.755873, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1693.1343162432313, "mean_wall_clock_sec": 22.87371075619012, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.4510415494441986, "step_time_p99": 22.517268266528845 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-07T11-21-07/matrix.json", "duration_sec": 36.969105, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1610.9927741345018, "mean_wall_clock_sec": 17.961225278675556, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.20964914560318, "step_time_p99": 8.152934350073338 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-07T11-21-44/matrix.json", "duration_sec": 32.113864, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2405.670928955078, "mean_wall_clock_sec": 19.414932087995112, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2405.670928955078, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-07T11-22-16/matrix.json", "duration_sec": 32.111861, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12509.872436523438, "mean_wall_clock_sec": 24.38283960428089, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-07T11-22-49/matrix.json", "duration_sec": 111.233925, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12829.043353907764, "mean_wall_clock_sec": 22.664334155619144, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-07T11-22-49/matrix.json", "duration_sec": 111.233925, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12793.103066273034, "mean_wall_clock_sec": 24.664582535624504, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-07T11-22-49/matrix.json", "duration_sec": 111.233925, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13446.725186891854, "mean_wall_clock_sec": 26.97819876577705, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-07T11-22-49/matrix.json", "duration_sec": 111.233925, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1808.4113223012537, "mean_wall_clock_sec": 27.38951192982495, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3225497677922249, "step_time_p99": 42.92700393125415 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-07T11-24-40/matrix.json", "duration_sec": 47.085426, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2156.8143852055073, "mean_wall_clock_sec": 27.334354016929865, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.83425460010767, "step_time_p99": 50.38199305534363 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-07T11-25-27/matrix.json", "duration_sec": 47.438631, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4391.5581703186035, "mean_wall_clock_sec": 34.76226708013564, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4391.5581703186035, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-07T11-26-14/matrix.json", "duration_sec": 47.789569, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 19841.625445522368, "mean_wall_clock_sec": 38.31003239471465, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-07T11-27-02/matrix.json", "duration_sec": 168.831818, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20033.45086798072, "mean_wall_clock_sec": 36.90028916671872, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-07T11-27-02/matrix.json", "duration_sec": 168.831818, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20100.765477865934, "mean_wall_clock_sec": 38.95767820253968, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-07T11-27-02/matrix.json", "duration_sec": 168.831818, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20922.367490828037, "mean_wall_clock_sec": 41.2393079418689, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-07T11-27-02/matrix.json", "duration_sec": 168.831818, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-08T11:20:12.448142+00:00", "build": { "amd_aorta_version": "0.2.2rc20260808", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31254345974", "head_sha": "0f2cb230223c53e468346d4e643a1236013d4146", "wheel_file": "amd_aorta-0.2.2rc20260808-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 255.36842085421085, "mean_wall_clock_sec": 5.224331057630479, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-08T11-10-46/matrix.json", "duration_sec": 16.589799, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 40.648039896041155, "mean_wall_clock_sec": 12.220068362541497, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1437514331191778, "decode_latency_ms": 1.0376853883386619, "tokens_per_sec": 3854.7328939496915, "decoded_tokens": 496.0, "step_time_p50": 37.96755336225033, "step_time_p99": 48.88582043349742, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-08T11-11-03/matrix.json", "duration_sec": 23.847378, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1702.5517271831632, "mean_wall_clock_sec": 22.792003027163446, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.416679471731186, "step_time_p99": 41.192007549107075 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-08T11-11-28/matrix.json", "duration_sec": 37.069937, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1637.3774274252355, "mean_wall_clock_sec": 18.134313449263573, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.206369958817959, "step_time_p99": 7.7793351002037525 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-08T11-12-05/matrix.json", "duration_sec": 32.263496, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2570.31831741333, "mean_wall_clock_sec": 20.29037509392947, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2570.31831741333, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-08T11-12-37/matrix.json", "duration_sec": 32.964384, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12471.64716757834, "mean_wall_clock_sec": 23.970574126578867, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-08T11-13-10/matrix.json", "duration_sec": 111.749435, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12911.29372175783, "mean_wall_clock_sec": 22.828755805268884, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-08T11-13-10/matrix.json", "duration_sec": 111.749435, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13052.776721306145, "mean_wall_clock_sec": 24.70194871444255, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-08T11-13-10/matrix.json", "duration_sec": 111.749435, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13616.249565966427, "mean_wall_clock_sec": 27.32786577194929, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-08T11-13-10/matrix.json", "duration_sec": 111.749435, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1869.1644994542003, "mean_wall_clock_sec": 27.230746378190815, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.5766657888889313, "step_time_p99": 36.45060667768121 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-08T11-15-02/matrix.json", "duration_sec": 47.243012, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2186.9459562003613, "mean_wall_clock_sec": 27.376723244786263, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.838954985141754, "step_time_p99": 51.51459874585271 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-08T11-15-49/matrix.json", "duration_sec": 47.3894, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4212.182998657227, "mean_wall_clock_sec": 33.62378488481045, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4212.182998657227, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-08T11-16-36/matrix.json", "duration_sec": 47.235684, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20421.875869855285, "mean_wall_clock_sec": 38.94358174968511, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-08T11-17-24/matrix.json", "duration_sec": 169.437799, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20318.7858434394, "mean_wall_clock_sec": 36.92532139457762, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-08T11-17-24/matrix.json", "duration_sec": 169.437799, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20124.71606209874, "mean_wall_clock_sec": 39.10689332522452, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-08T11-17-24/matrix.json", "duration_sec": 169.437799, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20895.95838263631, "mean_wall_clock_sec": 41.11047344096005, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-08T11-17-24/matrix.json", "duration_sec": 169.437799, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-09T11:20:19.787707+00:00", "build": { "amd_aorta_version": "0.2.2rc20260809", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31310115635", "head_sha": "0f2cb230223c53e468346d4e643a1236013d4146", "wheel_file": "amd_aorta-0.2.2rc20260809-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 259.0760299935937, "mean_wall_clock_sec": 5.239308069460094, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-09T11-10-53/matrix.json", "duration_sec": 16.590531, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 39.5978563465178, "mean_wall_clock_sec": 12.229454590938985, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.129976473748684, "decode_latency_ms": 1.0090964741163677, "tokens_per_sec": 3963.9421032589253, "decoded_tokens": 496.0, "step_time_p50": 37.04320499673486, "step_time_p99": 47.500328551977866, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-09T11-11-10/matrix.json", "duration_sec": 23.998047, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1721.3161813560873, "mean_wall_clock_sec": 22.934926605783403, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.6315160319209099, "step_time_p99": 2.922531086951494 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-09T11-11-35/matrix.json", "duration_sec": 37.023414, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1669.9020485393703, "mean_wall_clock_sec": 18.227149753831327, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.700739428400993, "step_time_p99": 56.58369418233633 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-09T11-12-12/matrix.json", "duration_sec": 32.313421, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2508.7972164154053, "mean_wall_clock_sec": 19.731439848430455, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2508.7972164154053, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-09T11-12-44/matrix.json", "duration_sec": 32.31526, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12848.138654604554, "mean_wall_clock_sec": 24.56810181029141, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-09T11-13-16/matrix.json", "duration_sec": 112.238392, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12867.975666187704, "mean_wall_clock_sec": 22.644155701622367, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-09T11-13-16/matrix.json", "duration_sec": 112.238392, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12816.606108099222, "mean_wall_clock_sec": 24.778915088623762, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-09T11-13-16/matrix.json", "duration_sec": 112.238392, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13835.68571601063, "mean_wall_clock_sec": 27.47404182050377, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-09T11-13-16/matrix.json", "duration_sec": 112.238392, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1793.6965790577233, "mean_wall_clock_sec": 26.743284991942346, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.35011225938797, "step_time_p99": 40.888639502227306 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-09T11-15-09/matrix.json", "duration_sec": 46.79058, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2178.575353231281, "mean_wall_clock_sec": 26.968474474735558, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.79318793118, "step_time_p99": 52.174547500908375 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-09T11-15-56/matrix.json", "duration_sec": 47.484842, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4236.792039871216, "mean_wall_clock_sec": 33.724637508392334, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4236.792039871216, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-09T11-16-43/matrix.json", "duration_sec": 47.083313, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20430.929178372025, "mean_wall_clock_sec": 39.2966170925647, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-09T11-17-30/matrix.json", "duration_sec": 170.386501, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20030.275216326118, "mean_wall_clock_sec": 36.572131807915866, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-09T11-17-30/matrix.json", "duration_sec": 170.386501, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20570.469581522048, "mean_wall_clock_sec": 39.522753960452974, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-09T11-17-30/matrix.json", "duration_sec": 170.386501, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 21258.195027709007, "mean_wall_clock_sec": 41.907571125775576, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-09T11-17-30/matrix.json", "duration_sec": 170.386501, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-10T11:35:03.122973+00:00", "build": { "amd_aorta_version": "0.2.2rc20260810", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31383376631", "head_sha": "b79ffc3bb7360a3b6c0f5a15397c512acba2c5fe", "wheel_file": "amd_aorta-0.2.2rc20260810-py3-none-any.whl" }, "summary": { "total": 16, "pass": 0, "fail": 0, "record": 16, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 263.1540531292558, "mean_wall_clock_sec": 5.218323276378214, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-10T11-25-39/matrix.json", "duration_sec": 16.493944, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 37.7242227550596, "mean_wall_clock_sec": 12.312535801902413, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.0789150837808847, "decode_latency_ms": 0.961465670937492, "tokens_per_sec": 4160.314945098079, "decoded_tokens": 496.0, "step_time_p50": 35.40200833231211, "step_time_p99": 45.0842841155827, "logits_checksum": 33904201.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-10T11-25-55/matrix.json", "duration_sec": 23.951004, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1662.0176599826664, "mean_wall_clock_sec": 22.964321916922927, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.9729789346456528, "step_time_p99": 3.3918134309351444 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-10T11-26-20/matrix.json", "duration_sec": 37.222887, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1569.4950660690665, "mean_wall_clock_sec": 17.918698858469725, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.758828811347485, "step_time_p99": 48.42123819515109 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-10T11-26-58/matrix.json", "duration_sec": 32.163926, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2410.573101043701, "mean_wall_clock_sec": 19.592885647900403, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2410.573101043701, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-10T11-27-30/matrix.json", "duration_sec": 32.26596, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12491.34769383818, "mean_wall_clock_sec": 24.340490957722068, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-10T11-28-02/matrix.json", "duration_sec": 110.73731, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12451.228047721088, "mean_wall_clock_sec": 22.245169366709888, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-10T11-28-02/matrix.json", "duration_sec": 110.73731, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 12602.386581711471, "mean_wall_clock_sec": 24.330416724085808, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-10T11-28-02/matrix.json", "duration_sec": 110.73731, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 13583.405612967908, "mean_wall_clock_sec": 27.147009733133018, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-10T11-28-02/matrix.json", "duration_sec": 110.73731, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 1711.0615279525518, "mean_wall_clock_sec": 26.81342104077339, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.4263913035392761, "step_time_p99": 43.12122793868184 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-10T11-29-53/matrix.json", "duration_sec": 46.785697, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 2171.4422691147774, "mean_wall_clock_sec": 27.894976002164185, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.887407973408699, "step_time_p99": 46.92627523094416 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-10T11-30-39/matrix.json", "duration_sec": 47.441946, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 4426.046705245972, "mean_wall_clock_sec": 34.59697254374623, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4426.046705245972, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-10T11-31-27/matrix.json", "duration_sec": 47.443339, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20921.22139595449, "mean_wall_clock_sec": 39.70283759199083, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-10T11-32-14/matrix.json", "duration_sec": 169.386437, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 19818.10911744833, "mean_wall_clock_sec": 36.36040620226413, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-10T11-32-14/matrix.json", "duration_sec": 169.386437, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 20149.469181895256, "mean_wall_clock_sec": 38.94213449489325, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-10T11-32-14/matrix.json", "duration_sec": 169.386437, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "record", "reasons": [ "no baseline (record-only)" ], "metrics": { "mean_step_time_ms": 21241.031749174, "mean_wall_clock_sec": 41.30899911560118, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-10T11-32-14/matrix.json", "duration_sec": 169.386437, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-11T11:28:27.910521+00:00", "build": { "amd_aorta_version": "0.2.2rc20260811", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31485940748", "head_sha": "2c9b36aaf4bfa1a012a91223f9f5794fd686e6f9", "wheel_file": "amd_aorta-0.2.2rc20260811-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 263.21482844650745, "mean_wall_clock_sec": 5.251191668212414, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-11T11-19-01/matrix.json", "duration_sec": 16.443571, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 41.7150545399636, "mean_wall_clock_sec": 12.117381162941456, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.156385987997055, "decode_latency_ms": 1.0614038910716772, "tokens_per_sec": 3768.593683938057, "decoded_tokens": 496.0, "step_time_p50": 39.30109040811658, "step_time_p99": 49.38843511976301, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-11T11-19-18/matrix.json", "duration_sec": 23.702264, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1699.7701791115105, "mean_wall_clock_sec": 23.001878977753222, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.481139101088047, "step_time_p99": 2.852336335927248 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-11T11-19-42/matrix.json", "duration_sec": 37.173244, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1664.3307788763195, "mean_wall_clock_sec": 18.21417643968016, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.759048439562321, "step_time_p99": 52.29593690484762 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-11T11-20-19/matrix.json", "duration_sec": 32.417018, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2563.9849185943604, "mean_wall_clock_sec": 20.178568560630083, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2563.9849185943604, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-11T11-20-52/matrix.json", "duration_sec": 32.914299, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12378.87912709266, "mean_wall_clock_sec": 23.76408983208239, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-11T11-21-25/matrix.json", "duration_sec": 111.872033, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13243.130915798247, "mean_wall_clock_sec": 23.248989993706346, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-11T11-21-25/matrix.json", "duration_sec": 111.872033, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12802.782418206334, "mean_wall_clock_sec": 24.681874771602452, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-11T11-21-25/matrix.json", "duration_sec": 111.872033, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13889.166696928442, "mean_wall_clock_sec": 27.341128583066165, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-11T11-21-25/matrix.json", "duration_sec": 111.872033, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1780.829481082037, "mean_wall_clock_sec": 26.900644360110164, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3932408764958382, "step_time_p99": 44.08884607255459 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-11T11-23-16/matrix.json", "duration_sec": 46.640512, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2166.909009916708, "mean_wall_clock_sec": 26.960168151184916, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 7.041523233056068, "step_time_p99": 53.16150130704045 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-11T11-24-03/matrix.json", "duration_sec": 47.28737, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 4390.193700790405, "mean_wall_clock_sec": 34.20897959638387, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4390.193700790405, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-11T11-24-50/matrix.json", "duration_sec": 47.482854, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20805.414685979486, "mean_wall_clock_sec": 39.3887635320425, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-11T11-25-38/matrix.json", "duration_sec": 170.486053, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20291.125737130642, "mean_wall_clock_sec": 37.1124569978565, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-11T11-25-38/matrix.json", "duration_sec": 170.486053, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20331.096881069243, "mean_wall_clock_sec": 38.52760707680136, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-11T11-25-38/matrix.json", "duration_sec": 170.486053, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 22151.924121193588, "mean_wall_clock_sec": 42.1372960601002, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-11T11-25-38/matrix.json", "duration_sec": 170.486053, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-12T11:29:44.056617+00:00", "build": { "amd_aorta_version": "0.2.2rc20260812", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31591245263", "head_sha": "261893abafd3b0f7db4ce519c615fc119c101948", "wheel_file": "amd_aorta-0.2.2rc20260812-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 253.12974397093058, "mean_wall_clock_sec": 5.22115916851908, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-12T11-20-21/matrix.json", "duration_sec": 16.440887, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.69972883351147, "mean_wall_clock_sec": 11.984144238755107, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1243245098739862, "decode_latency_ms": 1.0058142605327791, "tokens_per_sec": 3976.8773986970546, "decoded_tokens": 496.0, "step_time_p50": 36.843274254351854, "step_time_p99": 48.44144762493669, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-12T11-20-37/matrix.json", "duration_sec": 23.497794, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1729.8864896874875, "mean_wall_clock_sec": 22.842162866145372, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.83976162225008, "step_time_p99": 3.403056748211384 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-12T11-21-01/matrix.json", "duration_sec": 36.922, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1607.1204815525562, "mean_wall_clock_sec": 17.7675923127681, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.890849210321903, "step_time_p99": 8.862950969487429 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-12T11-21-39/matrix.json", "duration_sec": 31.965347, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2478.2472133636475, "mean_wall_clock_sec": 19.867794641293585, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2478.2472133636475, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-12T11-22-10/matrix.json", "duration_sec": 32.615327, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12414.160341024399, "mean_wall_clock_sec": 23.608094963245094, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-12T11-22-43/matrix.json", "duration_sec": 111.687377, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13281.55284654349, "mean_wall_clock_sec": 22.943328025750816, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-12T11-22-43/matrix.json", "duration_sec": 111.687377, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12839.393649250269, "mean_wall_clock_sec": 24.52779197320342, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-12T11-22-43/matrix.json", "duration_sec": 111.687377, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13896.536908112466, "mean_wall_clock_sec": 27.3889137590304, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-12T11-22-43/matrix.json", "duration_sec": 111.687377, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1726.850841427222, "mean_wall_clock_sec": 27.086075413040817, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3945987448096275, "step_time_p99": 3.277400489896536 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-12T11-24-35/matrix.json", "duration_sec": 46.840833, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2027.013276470825, "mean_wall_clock_sec": 26.98370935767889, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.869407370686531, "step_time_p99": 57.61626102030277 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-12T11-25-22/matrix.json", "duration_sec": 46.786914, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 4297.474718093872, "mean_wall_clock_sec": 33.57293556071818, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4297.474718093872, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-12T11-26-08/matrix.json", "duration_sec": 46.784808, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20784.364216029644, "mean_wall_clock_sec": 39.41499579511583, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-12T11-26-55/matrix.json", "duration_sec": 169.497466, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20037.78929822147, "mean_wall_clock_sec": 36.75146439950913, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-12T11-26-55/matrix.json", "duration_sec": 169.497466, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20380.165077745914, "mean_wall_clock_sec": 38.7182161193341, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-12T11-26-55/matrix.json", "duration_sec": 169.497466, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20970.74620053172, "mean_wall_clock_sec": 40.98928700387478, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-12T11-26-55/matrix.json", "duration_sec": 169.497466, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-13T11:29:42.392487+00:00", "build": { "amd_aorta_version": "0.2.2rc20260813", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31694891650", "head_sha": "f1660c492df6c68bf6750582652688ebbea181c6", "wheel_file": "amd_aorta-0.2.2rc20260813-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 262.52032071352005, "mean_wall_clock_sec": 5.266971984878182, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-13T11-20-24/matrix.json", "duration_sec": 16.540746, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 40.974187199026346, "mean_wall_clock_sec": 12.309063811786473, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1496341321617365, "decode_latency_ms": 1.0433271677503664, "tokens_per_sec": 3833.8884710774328, "decoded_tokens": 496.0, "step_time_p50": 38.17529650405049, "step_time_p99": 49.541148515418165, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-13T11-20-40/matrix.json", "duration_sec": 23.902669, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1604.4526842888445, "mean_wall_clock_sec": 22.11495865508914, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3838708400726318, "step_time_p99": 47.28996587917209 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-13T11-21-05/matrix.json", "duration_sec": 36.215439, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1555.4604092612863, "mean_wall_clock_sec": 17.777816079556942, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.841730326414108, "step_time_p99": 53.83259208872914 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-13T11-21-41/matrix.json", "duration_sec": 31.91191, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2404.4296741485596, "mean_wall_clock_sec": 19.22380622290075, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2404.4296741485596, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-13T11-22-13/matrix.json", "duration_sec": 31.858189, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12272.61546254158, "mean_wall_clock_sec": 23.645540471188724, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-13T11-22-45/matrix.json", "duration_sec": 110.029803, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12611.946281045675, "mean_wall_clock_sec": 22.14676600974053, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-13T11-22-45/matrix.json", "duration_sec": 110.029803, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12639.095018617809, "mean_wall_clock_sec": 24.097687176428735, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-13T11-22-45/matrix.json", "duration_sec": 110.029803, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13698.015987873077, "mean_wall_clock_sec": 27.29716850630939, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-13T11-22-45/matrix.json", "duration_sec": 110.029803, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1693.3306944556534, "mean_wall_clock_sec": 26.963683175854385, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.377701759338379, "step_time_p99": 44.54836254939437 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-13T11-24-35/matrix.json", "duration_sec": 46.682741, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2084.5262557268143, "mean_wall_clock_sec": 27.153035843744874, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 7.117412984371185, "step_time_p99": 48.87508874759078 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-13T11-25-22/matrix.json", "duration_sec": 46.631783, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 4228.075647354126, "mean_wall_clock_sec": 33.208609224297106, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4228.075647354126, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-13T11-26-08/matrix.json", "duration_sec": 46.432587, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20183.570676483214, "mean_wall_clock_sec": 38.40616456512362, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-13T11-26-55/matrix.json", "duration_sec": 168.213407, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20524.127314798534, "mean_wall_clock_sec": 37.28278385847807, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-13T11-26-55/matrix.json", "duration_sec": 168.213407, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 19979.209180921316, "mean_wall_clock_sec": 38.31846216507256, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-13T11-26-55/matrix.json", "duration_sec": 168.213407, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20650.496271438897, "mean_wall_clock_sec": 40.82087626494467, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-13T11-26-55/matrix.json", "duration_sec": 168.213407, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-14T11:27:27.877473+00:00", "build": { "amd_aorta_version": "0.2.2rc20260814", "torch": "2.9.1+rocm7.2.0.git7e1940d4", "hip": "7.2.26015-fc0010cf6a", "rocm": "7.2.0", "upstream_run_id": "31795583348", "head_sha": "68c980aa449f656a870923831f63a8aa34c145a3", "wheel_file": "amd_aorta-0.2.2rc20260814-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 260.09145472198725, "mean_wall_clock_sec": 5.221863987855613, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-14T11-18-10/matrix.json", "duration_sec": 16.440987, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 40.023294277489185, "mean_wall_clock_sec": 12.281000269576907, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1529845651239157, "decode_latency_ms": 1.0219816674268054, "tokens_per_sec": 3913.9645333085014, "decoded_tokens": 496.0, "step_time_p50": 37.42962284013629, "step_time_p99": 48.07951090857386, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-14T11-18-26/matrix.json", "duration_sec": 23.803201, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1613.2236446719617, "mean_wall_clock_sec": 22.189679600298405, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.5153000131249428, "step_time_p99": 2.741127572953701 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-14T11-18-51/matrix.json", "duration_sec": 36.318821, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1540.6226278282702, "mean_wall_clock_sec": 17.572337930090725, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.801075302064419, "step_time_p99": 8.759378921240568 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-14T11-19-27/matrix.json", "duration_sec": 31.463661, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2429.545259475708, "mean_wall_clock_sec": 19.291586606763303, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2429.545259475708, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-14T11-19-59/matrix.json", "duration_sec": 32.062, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12251.352391205728, "mean_wall_clock_sec": 23.532283797860146, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-14T11-20-31/matrix.json", "duration_sec": 109.732421, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12399.84487183392, "mean_wall_clock_sec": 22.275183488614857, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-14T11-20-31/matrix.json", "duration_sec": 109.732421, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12245.902427472174, "mean_wall_clock_sec": 24.068216983228922, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-14T11-20-31/matrix.json", "duration_sec": 109.732421, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 13343.778277747333, "mean_wall_clock_sec": 27.0389939295128, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-14T11-20-31/matrix.json", "duration_sec": 109.732421, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1713.9373333193362, "mean_wall_clock_sec": 26.633500647731125, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3037025928497314, "step_time_p99": 38.99399098008871 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-14T11-22-21/matrix.json", "duration_sec": 46.42896, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2073.587150545791, "mean_wall_clock_sec": 26.521744166500866, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.748643703758717, "step_time_p99": 48.88037519529462 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-14T11-23-07/matrix.json", "duration_sec": 46.63249, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 4305.575084686279, "mean_wall_clock_sec": 33.71009629312903, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 4305.575084686279, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-14T11-23-54/matrix.json", "duration_sec": 46.984833, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 19629.427135922015, "mean_wall_clock_sec": 37.40280138235539, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-14T11-24-41/matrix.json", "duration_sec": 167.579774, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20361.940055154264, "mean_wall_clock_sec": 36.855983689427376, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-14T11-24-41/matrix.json", "duration_sec": 167.579774, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20310.37274748087, "mean_wall_clock_sec": 39.12545967660844, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-14T11-24-41/matrix.json", "duration_sec": 167.579774, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 20262.816313654184, "mean_wall_clock_sec": 40.73439052514732, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-14T11-24-41/matrix.json", "duration_sec": 167.579774, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-15T11:15:12.128617+00:00", "build": { "amd_aorta_version": "0.2.2rc20260815", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "31881150246", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260815-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 240.73858931660652, "mean_wall_clock_sec": 5.267860027961433, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-15T11-06-42/matrix.json", "duration_sec": 16.687137, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.339321199804544, "mean_wall_clock_sec": 12.115215465426445, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.122721703723073, "decode_latency_ms": 1.0186246851639402, "tokens_per_sec": 3926.8634053927617, "decoded_tokens": 496.0, "step_time_p50": 37.392543628811836, "step_time_p99": 45.62636723741888, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-15T11-06-59/matrix.json", "duration_sec": 23.998184, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1499.2294816765934, "mean_wall_clock_sec": 20.434079174883664, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.5484662726521492, "step_time_p99": 40.42294502258301 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-15T11-07-24/matrix.json", "duration_sec": 34.509626, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1363.4722467977554, "mean_wall_clock_sec": 16.152988638728857, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.201250642538071, "step_time_p99": 46.521417163312435 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-15T11-07-58/matrix.json", "duration_sec": 30.359536, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2126.310968399048, "mean_wall_clock_sec": 17.753313584253192, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2126.310968399048, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-15T11-08-28/matrix.json", "duration_sec": 30.65532, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11084.864099510014, "mean_wall_clock_sec": 21.930390063673258, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-15T11-08-59/matrix.json", "duration_sec": 101.154989, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10986.420976929367, "mean_wall_clock_sec": 20.464767917990685, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-15T11-08-59/matrix.json", "duration_sec": 101.154989, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10788.644913583994, "mean_wall_clock_sec": 21.494885561987758, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-15T11-08-59/matrix.json", "duration_sec": 101.154989, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11776.721992529929, "mean_wall_clock_sec": 24.27548613306135, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-15T11-08-59/matrix.json", "duration_sec": 101.154989, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1490.8669118303806, "mean_wall_clock_sec": 21.936897632665932, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.4183791354298592, "step_time_p99": 36.09136715531349 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-15T11-10-40/matrix.json", "duration_sec": 41.52674, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1712.1146798599511, "mean_wall_clock_sec": 22.035746741108596, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.432509981095791, "step_time_p99": 42.804774809628725 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-15T11-11-22/matrix.json", "duration_sec": 41.323072, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3467.294216156006, "mean_wall_clock_sec": 29.452316823415458, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3467.294216156006, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-15T11-12-03/matrix.json", "duration_sec": 42.83061, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15953.574010170996, "mean_wall_clock_sec": 33.051273537799716, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-15T11-12-46/matrix.json", "duration_sec": 146.834406, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15710.607746616006, "mean_wall_clock_sec": 31.475311533547938, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-15T11-12-46/matrix.json", "duration_sec": 146.834406, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15696.602526120842, "mean_wall_clock_sec": 33.45843681227416, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-15T11-12-46/matrix.json", "duration_sec": 146.834406, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16391.406290233135, "mean_wall_clock_sec": 35.33252447191626, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-15T11-12-46/matrix.json", "duration_sec": 146.834406, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-16T11:14:55.928158+00:00", "build": { "amd_aorta_version": "0.2.2rc20260816", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "31943385514", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260816-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 240.50129298120737, "mean_wall_clock_sec": 5.297880647704005, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-16T11-06-23/matrix.json", "duration_sec": 16.73894, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.8774205930531, "mean_wall_clock_sec": 12.166678914800286, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1124599259346724, "decode_latency_ms": 1.0100671450697607, "tokens_per_sec": 3960.1327689197715, "decoded_tokens": 496.0, "step_time_p50": 37.036873400211334, "step_time_p99": 44.80732929892837, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-16T11-06-39/matrix.json", "duration_sec": 23.949941, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1502.5542215444148, "mean_wall_clock_sec": 20.508043660782278, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.4607692137360573, "step_time_p99": 44.50176874175668 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-16T11-07-04/matrix.json", "duration_sec": 34.561007, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1368.8209091778845, "mean_wall_clock_sec": 16.266320697031915, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.131202913820744, "step_time_p99": 43.89042450115085 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-16T11-07-39/matrix.json", "duration_sec": 30.411176, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2113.125705718994, "mean_wall_clock_sec": 17.734310601837933, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2113.125705718994, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-16T11-08-09/matrix.json", "duration_sec": 30.455536, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10816.860957071185, "mean_wall_clock_sec": 21.62442221585661, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-16T11-08-40/matrix.json", "duration_sec": 103.31191, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10742.653512395918, "mean_wall_clock_sec": 20.191979269497097, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-16T11-08-40/matrix.json", "duration_sec": 103.31191, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10941.272908821702, "mean_wall_clock_sec": 23.874935754574835, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-16T11-08-40/matrix.json", "duration_sec": 103.31191, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11991.375438869, "mean_wall_clock_sec": 24.57630441803485, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-16T11-08-40/matrix.json", "duration_sec": 103.31191, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1485.5729832779616, "mean_wall_clock_sec": 21.988396298140287, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3215932995080948, "step_time_p99": 35.0089680776 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-16T11-10-23/matrix.json", "duration_sec": 42.074876, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1753.9817448705435, "mean_wall_clock_sec": 22.65231193229556, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.718832068145275, "step_time_p99": 39.79077950119972 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-16T11-11-05/matrix.json", "duration_sec": 42.285602, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3333.3354473114014, "mean_wall_clock_sec": 29.068887849338353, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3333.3354473114014, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-16T11-11-47/matrix.json", "duration_sec": 42.484442, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16302.446614019573, "mean_wall_clock_sec": 33.36399857327342, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-16T11-12-30/matrix.json", "duration_sec": 146.70585, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15887.534871697426, "mean_wall_clock_sec": 31.569105656817555, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-16T11-12-30/matrix.json", "duration_sec": 146.70585, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16008.867741562426, "mean_wall_clock_sec": 33.43999211490154, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-16T11-12-30/matrix.json", "duration_sec": 146.70585, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16398.72939325869, "mean_wall_clock_sec": 34.99136808887124, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-16T11-12-30/matrix.json", "duration_sec": 146.70585, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-17T11:16:59.182018+00:00", "build": { "amd_aorta_version": "0.2.2rc20260817", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32023392322", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260817-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 239.7684333845973, "mean_wall_clock_sec": 5.30011729337275, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-17T11-08-30/matrix.json", "duration_sec": 16.734976, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.36372634395957, "mean_wall_clock_sec": 12.013368157669902, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1055618524551392, "decode_latency_ms": 0.9936654654842231, "tokens_per_sec": 4025.4996665812073, "decoded_tokens": 496.0, "step_time_p50": 36.485247779637575, "step_time_p99": 44.27859482355415, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-17T11-08-46/matrix.json", "duration_sec": 23.798586, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1439.8597134277225, "mean_wall_clock_sec": 20.131904423236847, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.2298347428441048, "step_time_p99": 2.7912862226366997 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-17T11-09-11/matrix.json", "duration_sec": 34.307152, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1372.093768324703, "mean_wall_clock_sec": 16.433698570355773, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.456355564296246, "step_time_p99": 46.593823209404945 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-17T11-09-46/matrix.json", "duration_sec": 30.455477, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2212.2395038604736, "mean_wall_clock_sec": 18.03349698614329, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2212.2395038604736, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-17T11-10-16/matrix.json", "duration_sec": 30.90424, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10897.381296381354, "mean_wall_clock_sec": 21.892534723505378, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-17T11-10-47/matrix.json", "duration_sec": 101.157274, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10737.723597325385, "mean_wall_clock_sec": 20.346767857670784, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-17T11-10-47/matrix.json", "duration_sec": 101.157274, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10737.11109161377, "mean_wall_clock_sec": 21.592149134725332, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-17T11-10-47/matrix.json", "duration_sec": 101.157274, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11762.136873789132, "mean_wall_clock_sec": 24.46489070635289, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-17T11-10-47/matrix.json", "duration_sec": 101.157274, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1477.0746666472405, "mean_wall_clock_sec": 22.096022200770676, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3599321246147156, "step_time_p99": 36.60381156951189 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-17T11-12-28/matrix.json", "duration_sec": 41.681196, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1732.2042719461024, "mean_wall_clock_sec": 22.415122106671333, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.22866116464138, "step_time_p99": 43.37807031348348 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-17T11-13-10/matrix.json", "duration_sec": 42.076741, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3365.7129764556885, "mean_wall_clock_sec": 28.789510077796876, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3365.7129764556885, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-17T11-13-52/matrix.json", "duration_sec": 42.078391, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15757.346913218498, "mean_wall_clock_sec": 32.96673267707229, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-17T11-14-34/matrix.json", "duration_sec": 145.891254, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15439.851799048483, "mean_wall_clock_sec": 31.195682109333575, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-17T11-14-34/matrix.json", "duration_sec": 145.891254, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15719.611783511937, "mean_wall_clock_sec": 32.51497147511691, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-17T11-14-34/matrix.json", "duration_sec": 145.891254, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16494.807681068778, "mean_wall_clock_sec": 35.50702884979546, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-17T11-14-34/matrix.json", "duration_sec": 145.891254, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-18T11:17:30.180618+00:00", "build": { "amd_aorta_version": "0.2.2rc20260818", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32130179193", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260818-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 237.30712942779064, "mean_wall_clock_sec": 5.312692791223526, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-18T11-08-59/matrix.json", "duration_sec": 16.738153, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.85857411660254, "mean_wall_clock_sec": 12.140414603985846, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1093143839389086, "decode_latency_ms": 1.0123304091393948, "tokens_per_sec": 3951.2791119260078, "decoded_tokens": 496.0, "step_time_p50": 37.05537086352706, "step_time_p99": 44.7054436430335, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-18T11-09-16/matrix.json", "duration_sec": 23.950281, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1493.3616607449949, "mean_wall_clock_sec": 20.512447622604668, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.43332127481699, "step_time_p99": 50.552514642477036 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-18T11-09-40/matrix.json", "duration_sec": 34.55997, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1371.6229568235576, "mean_wall_clock_sec": 16.311638872139156, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.02720507979393, "step_time_p99": 9.344298671931028 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-18T11-10-15/matrix.json", "duration_sec": 30.560699, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2187.6026153564453, "mean_wall_clock_sec": 18.073920296505094, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2187.6026153564453, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-18T11-10-46/matrix.json", "duration_sec": 31.109267, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10914.216486737132, "mean_wall_clock_sec": 21.574545877054334, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-18T11-11-17/matrix.json", "duration_sec": 101.862672, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11002.831120975316, "mean_wall_clock_sec": 20.310319646261632, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-18T11-11-17/matrix.json", "duration_sec": 101.862672, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11349.35596678406, "mean_wall_clock_sec": 22.552851132117212, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-18T11-11-17/matrix.json", "duration_sec": 101.862672, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 12196.874346584082, "mean_wall_clock_sec": 24.56123543344438, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-18T11-11-17/matrix.json", "duration_sec": 101.862672, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1508.1140336114913, "mean_wall_clock_sec": 21.95547462441027, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.2852037325501442, "step_time_p99": 33.9847182109952 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-18T11-12-59/matrix.json", "duration_sec": 42.234594, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1724.9237794894725, "mean_wall_clock_sec": 22.042927011847496, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.397608667612076, "step_time_p99": 38.60194990411401 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-18T11-13-41/matrix.json", "duration_sec": 41.73374, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3341.372776031494, "mean_wall_clock_sec": 29.127501657232642, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3341.372776031494, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-18T11-14-23/matrix.json", "duration_sec": 42.730748, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16046.734469011426, "mean_wall_clock_sec": 33.11013107281178, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-18T11-15-05/matrix.json", "duration_sec": 145.461533, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15339.322070591152, "mean_wall_clock_sec": 31.068319760262966, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-18T11-15-05/matrix.json", "duration_sec": 145.461533, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15782.920946367085, "mean_wall_clock_sec": 32.89442281797528, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-18T11-15-05/matrix.json", "duration_sec": 145.461533, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16730.27839884162, "mean_wall_clock_sec": 35.17455215752125, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-18T11-15-05/matrix.json", "duration_sec": 145.461533, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-19T11:17:30.425460+00:00", "build": { "amd_aorta_version": "0.2.2rc20260819", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32246029090", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260819-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 240.1866503059864, "mean_wall_clock_sec": 5.30320449732244, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-19T11-09-00/matrix.json", "duration_sec": 16.738565, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.490268798545, "mean_wall_clock_sec": 12.143859754316509, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.125416485592723, "decode_latency_ms": 1.025620254597837, "tokens_per_sec": 3900.0789834912803, "decoded_tokens": 496.0, "step_time_p50": 37.60480089113116, "step_time_p99": 45.225091557949774, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-19T11-09-17/matrix.json", "duration_sec": 23.900109, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1502.6451568119228, "mean_wall_clock_sec": 20.513991434127092, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3942299410700798, "step_time_p99": 41.11238034442067 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-19T11-09-42/matrix.json", "duration_sec": 34.664071, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1354.4781277887523, "mean_wall_clock_sec": 16.236251724883914, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.192100398242474, "step_time_p99": 10.219727009534836 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-19T11-10-16/matrix.json", "duration_sec": 30.406431, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2204.883289337158, "mean_wall_clock_sec": 18.208185829222202, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2204.883289337158, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-19T11-10-47/matrix.json", "duration_sec": 31.3591, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11137.719267979264, "mean_wall_clock_sec": 21.949199311435223, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-19T11-11-18/matrix.json", "duration_sec": 101.213096, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10806.022743694484, "mean_wall_clock_sec": 20.465099283494055, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-19T11-11-18/matrix.json", "duration_sec": 101.213096, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10878.705536946654, "mean_wall_clock_sec": 21.790067163296044, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-19T11-11-18/matrix.json", "duration_sec": 101.213096, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11936.285804957151, "mean_wall_clock_sec": 24.133593225851655, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-19T11-11-18/matrix.json", "duration_sec": 101.213096, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1489.8519492708147, "mean_wall_clock_sec": 22.138302056118846, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3103019446134567, "step_time_p99": 38.96589966490865 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-19T11-12-59/matrix.json", "duration_sec": 41.732306, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1692.5367140211165, "mean_wall_clock_sec": 22.04865096323192, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.56251423060894, "step_time_p99": 42.23327808082104 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-19T11-13-41/matrix.json", "duration_sec": 41.481527, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3450.0006675720215, "mean_wall_clock_sec": 29.266516911797225, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3450.0006675720215, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-19T11-14-23/matrix.json", "duration_sec": 42.427283, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15603.942709043622, "mean_wall_clock_sec": 32.35093581024557, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-19T11-15-05/matrix.json", "duration_sec": 146.046704, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15762.533314526081, "mean_wall_clock_sec": 31.370761555619538, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-19T11-15-05/matrix.json", "duration_sec": 146.046704, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15544.399045407772, "mean_wall_clock_sec": 32.66600726451725, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-19T11-15-05/matrix.json", "duration_sec": 146.046704, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16806.145837530494, "mean_wall_clock_sec": 35.4277029633522, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-19T11-15-05/matrix.json", "duration_sec": 146.046704, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-20T11:17:47.546673+00:00", "build": { "amd_aorta_version": "0.2.2rc20260820", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32362312082", "head_sha": "b4be48fdd6dd7e12056a6141c93377feed9f1688", "wheel_file": "amd_aorta-0.2.2rc20260820-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 242.52371862530708, "mean_wall_clock_sec": 5.307661429978907, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-20T11-09-20/matrix.json", "duration_sec": 16.737702, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.656848318874836, "mean_wall_clock_sec": 12.133294523693621, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1051672045141459, "decode_latency_ms": 0.9832409390759084, "tokens_per_sec": 4068.178857319926, "decoded_tokens": 496.0, "step_time_p50": 35.857928451150656, "step_time_p99": 43.38063228875398, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-20T11-09-37/matrix.json", "duration_sec": 23.948753, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1445.1553565450013, "mean_wall_clock_sec": 20.175131743773818, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.2127319350838661, "step_time_p99": 39.17701169848442 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-20T11-10-02/matrix.json", "duration_sec": 34.459106, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1370.3240603208542, "mean_wall_clock_sec": 16.34715194068849, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.843785755336285, "step_time_p99": 9.233236070722342 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-20T11-10-36/matrix.json", "duration_sec": 30.455531, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2124.5278358459473, "mean_wall_clock_sec": 17.878547692671418, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2124.5278358459473, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-20T11-11-07/matrix.json", "duration_sec": 30.657539, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10958.436701446772, "mean_wall_clock_sec": 21.676840342581272, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-20T11-11-37/matrix.json", "duration_sec": 100.612754, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10712.271525524557, "mean_wall_clock_sec": 20.220503142103553, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-20T11-11-37/matrix.json", "duration_sec": 100.612754, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10771.862097084522, "mean_wall_clock_sec": 21.52925801463425, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-20T11-11-37/matrix.json", "duration_sec": 100.612754, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11660.74616741389, "mean_wall_clock_sec": 24.219774610362947, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-20T11-11-37/matrix.json", "duration_sec": 100.612754, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1490.0067024864256, "mean_wall_clock_sec": 22.13639580179006, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3094302266836166, "step_time_p99": 17.94599127024412 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-20T11-13-18/matrix.json", "duration_sec": 41.529999, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1744.9835669249296, "mean_wall_clock_sec": 22.33746588975191, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.498317234218121, "step_time_p99": 40.812471471726894 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-20T11-14-00/matrix.json", "duration_sec": 41.732551, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3462.02449798584, "mean_wall_clock_sec": 29.39775315206498, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3462.02449798584, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-20T11-14-41/matrix.json", "duration_sec": 42.431439, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16304.243920370936, "mean_wall_clock_sec": 33.05256715230644, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-20T11-15-24/matrix.json", "duration_sec": 144.392712, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15312.055619433522, "mean_wall_clock_sec": 30.58181604463607, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-20T11-15-24/matrix.json", "duration_sec": 144.392712, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15737.359289079905, "mean_wall_clock_sec": 32.92032269947231, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-20T11-15-24/matrix.json", "duration_sec": 144.392712, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16215.701598674059, "mean_wall_clock_sec": 34.9694653628394, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-20T11-15-24/matrix.json", "duration_sec": 144.392712, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-21T11:18:34.884626+00:00", "build": { "amd_aorta_version": "0.2.2rc20260821", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32475806192", "head_sha": "78d1ae686dc3a786e8cfdb1216efc4b7516c8896", "wheel_file": "amd_aorta-0.2.2rc20260821-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 236.77177261561155, "mean_wall_clock_sec": 5.271073141135275, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-21T11-10-08/matrix.json", "duration_sec": 16.691098, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.34452393837273, "mean_wall_clock_sec": 11.704970478080213, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.0886297095566988, "decode_latency_ms": 0.975183654396284, "tokens_per_sec": 4101.791474833853, "decoded_tokens": 496.0, "step_time_p50": 35.52645957097411, "step_time_p99": 42.88394795730709, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-21T11-10-24/matrix.json", "duration_sec": 23.401112, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1461.0238058958203, "mean_wall_clock_sec": 20.15646764356643, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3181120157241821, "step_time_p99": 40.01021035015583 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-21T11-10-49/matrix.json", "duration_sec": 34.26503, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1339.0891801100224, "mean_wall_clock_sec": 16.041489583440125, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.115712225437164, "step_time_p99": 20.863676816225052 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-21T11-11-23/matrix.json", "duration_sec": 30.210089, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2106.159543991089, "mean_wall_clock_sec": 17.74161039572209, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2106.159543991089, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-21T11-11-53/matrix.json", "duration_sec": 30.810917, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10667.714688926935, "mean_wall_clock_sec": 21.50011446606368, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-21T11-12-24/matrix.json", "duration_sec": 99.961886, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10549.829429015517, "mean_wall_clock_sec": 20.113333601504564, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-21T11-12-24/matrix.json", "duration_sec": 99.961886, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10622.549451887608, "mean_wall_clock_sec": 21.284804445691407, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-21T11-12-24/matrix.json", "duration_sec": 99.961886, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11481.46734572947, "mean_wall_clock_sec": 24.106316274031997, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-21T11-12-24/matrix.json", "duration_sec": 99.961886, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1472.739600110799, "mean_wall_clock_sec": 21.726381336338818, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.4641694724559784, "step_time_p99": 33.4595188125968 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-21T11-14-04/matrix.json", "duration_sec": 41.828602, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1778.1954128295183, "mean_wall_clock_sec": 22.463975781574845, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.690130569040775, "step_time_p99": 48.12376908957958 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-21T11-14-46/matrix.json", "duration_sec": 42.28159, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3286.0787868499756, "mean_wall_clock_sec": 28.83356460928917, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3286.0787868499756, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-21T11-15-28/matrix.json", "duration_sec": 41.984681, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16260.074535384774, "mean_wall_clock_sec": 33.10636247508228, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-21T11-16-10/matrix.json", "duration_sec": 145.387451, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15507.148738019168, "mean_wall_clock_sec": 30.953342811204493, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-21T11-16-10/matrix.json", "duration_sec": 145.387451, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16034.883397631347, "mean_wall_clock_sec": 33.05386161617935, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-21T11-16-10/matrix.json", "duration_sec": 145.387451, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16437.335273250937, "mean_wall_clock_sec": 35.29415332060307, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-21T11-16-10/matrix.json", "duration_sec": 145.387451, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-22T11:16:00.720522+00:00", "build": { "amd_aorta_version": "0.2.2rc20260822", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32569358944", "head_sha": "78d1ae686dc3a786e8cfdb1216efc4b7516c8896", "wheel_file": "amd_aorta-0.2.2rc20260822-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 235.99149473011494, "mean_wall_clock_sec": 5.285746136680245, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-22T11-07-34/matrix.json", "duration_sec": 16.686753, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.77181310579181, "mean_wall_clock_sec": 11.746343202888966, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1047336738556623, "decode_latency_ms": 0.9816139110273892, "tokens_per_sec": 4074.9218761717316, "decoded_tokens": 496.0, "step_time_p50": 35.979748237878084, "step_time_p99": 43.460513101890676, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-22T11-07-50/matrix.json", "duration_sec": 23.548511, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1445.243097608909, "mean_wall_clock_sec": 20.046109978109598, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.2733330950140953, "step_time_p99": 38.34832465276122 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-22T11-08-15/matrix.json", "duration_sec": 34.019368, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1352.7289365883917, "mean_wall_clock_sec": 16.150250118225813, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 5.901265889406204, "step_time_p99": 7.750653829425573 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-22T11-08-49/matrix.json", "duration_sec": 30.305949, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2103.1381607055664, "mean_wall_clock_sec": 17.73380667809397, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2103.1381607055664, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-22T11-09-19/matrix.json", "duration_sec": 30.708528, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10696.076097898185, "mean_wall_clock_sec": 21.615023902617395, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-22T11-09-50/matrix.json", "duration_sec": 100.406164, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10504.469817504287, "mean_wall_clock_sec": 19.832773745059967, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-22T11-09-50/matrix.json", "duration_sec": 100.406164, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10850.514655001462, "mean_wall_clock_sec": 21.52624668367207, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-22T11-09-50/matrix.json", "duration_sec": 100.406164, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11528.928031213582, "mean_wall_clock_sec": 24.253576095215976, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-22T11-09-50/matrix.json", "duration_sec": 100.406164, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1492.4605924170464, "mean_wall_clock_sec": 21.737953750416636, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.32015161216259, "step_time_p99": 37.89330815896392 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-22T11-11-30/matrix.json", "duration_sec": 41.676927, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1696.7130601406097, "mean_wall_clock_sec": 22.30385959148407, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.466853432357311, "step_time_p99": 45.75867982581258 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-22T11-12-12/matrix.json", "duration_sec": 41.57855, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3282.3338508605957, "mean_wall_clock_sec": 28.62143244035542, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3282.3338508605957, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-22T11-12-54/matrix.json", "duration_sec": 41.92532, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15709.436773322523, "mean_wall_clock_sec": 33.10877677612007, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-22T11-13-36/matrix.json", "duration_sec": 145.698408, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15561.73073593527, "mean_wall_clock_sec": 31.18131491728127, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-22T11-13-36/matrix.json", "duration_sec": 145.698408, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15662.5139368698, "mean_wall_clock_sec": 32.95006274804473, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-22T11-13-36/matrix.json", "duration_sec": 145.698408, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16182.085602544248, "mean_wall_clock_sec": 35.027068892493844, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-22T11-13-36/matrix.json", "duration_sec": 145.698408, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-23T11:15:45.192908+00:00", "build": { "amd_aorta_version": "0.2.2rc20260823", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32635578383", "head_sha": "78d1ae686dc3a786e8cfdb1216efc4b7516c8896", "wheel_file": "amd_aorta-0.2.2rc20260823-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 235.57757306843996, "mean_wall_clock_sec": 5.28454892616719, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-23T11-07-19/matrix.json", "duration_sec": 16.784715, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.77835564687848, "mean_wall_clock_sec": 11.75614975951612, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1024244595319033, "decode_latency_ms": 0.9877110621140849, "tokens_per_sec": 4049.7673392848797, "decoded_tokens": 496.0, "step_time_p50": 36.12035745754838, "step_time_p99": 43.17396068014204, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-23T11-07-35/matrix.json", "duration_sec": 23.443033, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1451.605943031609, "mean_wall_clock_sec": 20.003838704898953, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3609016314148903, "step_time_p99": 39.129668194800615 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-23T11-08-00/matrix.json", "duration_sec": 34.261082, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1348.0920889414847, "mean_wall_clock_sec": 16.189119055867195, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.6862404346466064, "step_time_p99": 42.55194431170821 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-23T11-08-34/matrix.json", "duration_sec": 30.052168, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2116.842222213745, "mean_wall_clock_sec": 17.71984703373164, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2116.842222213745, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-23T11-09-04/matrix.json", "duration_sec": 30.559211, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10689.121856354177, "mean_wall_clock_sec": 21.420121118426323, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-23T11-09-35/matrix.json", "duration_sec": 99.761717, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10573.184348642826, "mean_wall_clock_sec": 20.05773936212063, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-23T11-09-35/matrix.json", "duration_sec": 99.761717, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10623.609968461096, "mean_wall_clock_sec": 21.34400148037821, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-23T11-09-35/matrix.json", "duration_sec": 99.761717, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11816.276564262807, "mean_wall_clock_sec": 23.985289704985917, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-23T11-09-35/matrix.json", "duration_sec": 99.761717, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1497.23028880544, "mean_wall_clock_sec": 22.17314261198044, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.4355499297380447, "step_time_p99": 37.1989624761045 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-23T11-11-15/matrix.json", "duration_sec": 41.625275, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1712.4668930191547, "mean_wall_clock_sec": 22.226136481389403, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.526464596390724, "step_time_p99": 40.286454390734434 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-23T11-11-56/matrix.json", "duration_sec": 41.627397, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3277.7560234069824, "mean_wall_clock_sec": 28.70671635121107, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3277.7560234069824, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-23T11-12-38/matrix.json", "duration_sec": 42.176358, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16026.883664540946, "mean_wall_clock_sec": 33.171266884543, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-23T11-13-20/matrix.json", "duration_sec": 145.93736, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15377.49883159995, "mean_wall_clock_sec": 31.159218755550683, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-23T11-13-20/matrix.json", "duration_sec": 145.93736, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15735.43594405055, "mean_wall_clock_sec": 33.10310480091721, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-23T11-13-20/matrix.json", "duration_sec": 145.93736, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16430.15513662249, "mean_wall_clock_sec": 35.3288896875456, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-23T11-13-20/matrix.json", "duration_sec": 145.93736, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-24T11:19:29.218725+00:00", "build": { "amd_aorta_version": "0.2.2rc20260824", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32720415574", "head_sha": "78d1ae686dc3a786e8cfdb1216efc4b7516c8896", "wheel_file": "amd_aorta-0.2.2rc20260824-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 235.1088859140873, "mean_wall_clock_sec": 5.280710445716977, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-24T11-11-02/matrix.json", "duration_sec": 16.689494, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.145149825140834, "mean_wall_clock_sec": 11.828732777386904, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1093963403254747, "decode_latency_ms": 0.9969734556732639, "tokens_per_sec": 4012.142928418058, "decoded_tokens": 496.0, "step_time_p50": 36.33986273780465, "step_time_p99": 43.85674966499209, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-24T11-11-18/matrix.json", "duration_sec": 23.552623, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1431.7125550005585, "mean_wall_clock_sec": 19.911318310536444, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3109631836414337, "step_time_p99": 31.422648280858994 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-24T11-11-43/matrix.json", "duration_sec": 33.917125, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1342.5399514380842, "mean_wall_clock_sec": 16.134763727895916, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 5.984813906252384, "step_time_p99": 7.704697567969561 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-24T11-12-17/matrix.json", "duration_sec": 30.206352, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2092.42205619812, "mean_wall_clock_sec": 17.731036848388612, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2092.42205619812, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-24T11-12-47/matrix.json", "duration_sec": 30.553493, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10703.114220872521, "mean_wall_clock_sec": 21.64579026773572, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-24T11-13-18/matrix.json", "duration_sec": 100.617779, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10587.933888658881, "mean_wall_clock_sec": 20.132082361727953, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-24T11-13-18/matrix.json", "duration_sec": 100.617779, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10702.603168785572, "mean_wall_clock_sec": 21.6460904981941, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-24T11-13-18/matrix.json", "duration_sec": 100.617779, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11530.061191879213, "mean_wall_clock_sec": 24.23475146573037, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-24T11-13-18/matrix.json", "duration_sec": 100.617779, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1457.5733812525868, "mean_wall_clock_sec": 22.15849117282778, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.4493893831968307, "step_time_p99": 7.933088634163141 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-24T11-14-58/matrix.json", "duration_sec": 41.477634, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1684.1763143893331, "mean_wall_clock_sec": 21.644453623332083, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.599421612918377, "step_time_p99": 8.488507270812988 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-24T11-15-40/matrix.json", "duration_sec": 41.635393, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3303.0216693878174, "mean_wall_clock_sec": 28.79090760741383, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3303.0216693878174, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-24T11-16-21/matrix.json", "duration_sec": 42.030858, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15635.411915369332, "mean_wall_clock_sec": 32.71335295494646, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-24T11-17-03/matrix.json", "duration_sec": 146.395513, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15903.412813320756, "mean_wall_clock_sec": 31.504857651889324, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-24T11-17-03/matrix.json", "duration_sec": 146.395513, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15747.073326259851, "mean_wall_clock_sec": 33.22967263497412, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-24T11-17-03/matrix.json", "duration_sec": 146.395513, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16487.315769307315, "mean_wall_clock_sec": 35.26979696471244, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-24T11-17-03/matrix.json", "duration_sec": 146.395513, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-25T11:18:23.108730+00:00", "build": { "amd_aorta_version": "0.2.2rc20260825", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32840754928", "head_sha": "905b1f9e3e16865eb1c6362e72846ffc391ec901", "wheel_file": "amd_aorta-0.2.2rc20260825-py3-none-any.whl" }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 233.87731052935123, "mean_wall_clock_sec": 5.29486408084631, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-25T11-09-55/matrix.json", "duration_sec": 16.737971, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.983795162290335, "mean_wall_clock_sec": 11.78519247751683, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1033846531063318, "decode_latency_ms": 0.9892773213646104, "tokens_per_sec": 4043.355602736748, "decoded_tokens": 496.0, "step_time_p50": 36.28780785948038, "step_time_p99": 43.61481433734297, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-25T11-10-12/matrix.json", "duration_sec": 23.550319, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1470.1172281056643, "mean_wall_clock_sec": 20.0408681249246, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.4535393565893173, "step_time_p99": 38.38996812701225 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-25T11-10-36/matrix.json", "duration_sec": 33.9572, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1340.7082387711853, "mean_wall_clock_sec": 15.97348370961845, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 5.944347940385342, "step_time_p99": 7.826467901468277 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-25T11-11-10/matrix.json", "duration_sec": 30.054891, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2128.8984298706055, "mean_wall_clock_sec": 17.75265687610954, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2128.8984298706055, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-25T11-11-40/matrix.json", "duration_sec": 30.755462, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10741.978913545609, "mean_wall_clock_sec": 21.733522053807974, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-25T11-12-11/matrix.json", "duration_sec": 101.304864, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10555.000026710331, "mean_wall_clock_sec": 20.05611891206354, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-25T11-12-11/matrix.json", "duration_sec": 101.304864, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10821.25598192215, "mean_wall_clock_sec": 21.668509896844625, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-25T11-12-11/matrix.json", "duration_sec": 101.304864, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11660.791556350887, "mean_wall_clock_sec": 24.897972996346653, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-25T11-12-11/matrix.json", "duration_sec": 101.304864, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1496.7627150472254, "mean_wall_clock_sec": 22.38318782299757, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.2840423732995987, "step_time_p99": 37.04970745369792 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-25T11-13-52/matrix.json", "duration_sec": 41.921631, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1717.3174179624766, "mean_wall_clock_sec": 22.019770198501647, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.787409074604511, "step_time_p99": 8.617286309599876 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-25T11-14-34/matrix.json", "duration_sec": 41.632986, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3337.961196899414, "mean_wall_clock_sec": 28.900380549952388, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3337.961196899414, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-25T11-15-16/matrix.json", "duration_sec": 42.373372, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16005.638150498271, "mean_wall_clock_sec": 32.629749132320285, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-25T11-15-58/matrix.json", "duration_sec": 145.53711, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15605.233798734844, "mean_wall_clock_sec": 31.156551755964756, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-25T11-15-58/matrix.json", "duration_sec": 145.53711, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15696.398474276066, "mean_wall_clock_sec": 32.955879838205874, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-25T11-15-58/matrix.json", "duration_sec": 145.53711, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16360.146817751229, "mean_wall_clock_sec": 35.20332309324294, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-25T11-15-58/matrix.json", "duration_sec": 145.53711, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-26T11:20:46.901256+00:00", "build": { "amd_aorta_version": "0.2.2rc20260826", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "32961928464", "head_sha": "bed2771d52ddc0d81cdf28ac176611815c2358c7", "wheel_file": "amd_aorta-0.2.2rc20260826-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 233.56152419000864, "mean_wall_clock_sec": 5.26520602684468, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-26T11-12-19/matrix.json", "duration_sec": 16.687507, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.01488031260669, "mean_wall_clock_sec": 11.730220522731543, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.112152123823762, "decode_latency_ms": 1.0134012423335546, "tokens_per_sec": 3947.1039040658943, "decoded_tokens": 496.0, "step_time_p50": 37.13461197912693, "step_time_p99": 44.994293544441454, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-26T11-12-36/matrix.json", "duration_sec": 23.547999, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1454.9814362544566, "mean_wall_clock_sec": 20.12070677895099, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3767611235380173, "step_time_p99": 51.09157998114824 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-26T11-13-00/matrix.json", "duration_sec": 34.167463, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1344.9321605730802, "mean_wall_clock_sec": 16.069662526249886, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.384397856891155, "step_time_p99": 43.16663848236203 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-26T11-13-34/matrix.json", "duration_sec": 30.00601, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2108.3744525909424, "mean_wall_clock_sec": 17.75359221547842, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2108.3744525909424, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-26T11-14-04/matrix.json", "duration_sec": 30.46365, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10742.759527638555, "mean_wall_clock_sec": 21.65778189152479, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-26T11-14-35/matrix.json", "duration_sec": 100.159645, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10510.915488004684, "mean_wall_clock_sec": 19.987364585511386, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-26T11-14-35/matrix.json", "duration_sec": 100.159645, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10546.705640852451, "mean_wall_clock_sec": 21.320317154750228, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-26T11-14-35/matrix.json", "duration_sec": 100.159645, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11776.912255212665, "mean_wall_clock_sec": 24.059703732840717, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-26T11-14-35/matrix.json", "duration_sec": 100.159645, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1495.6245764624327, "mean_wall_clock_sec": 22.233465961180627, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.2725545093417168, "step_time_p99": 2.9819330386817455 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-26T11-16-15/matrix.json", "duration_sec": 41.485915, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1720.6108225509524, "mean_wall_clock_sec": 21.885590568184853, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.725982762873173, "step_time_p99": 47.88481030613184 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-26T11-16-57/matrix.json", "duration_sec": 41.925097, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3294.4380283355713, "mean_wall_clock_sec": 29.121033808216453, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3294.4380283355713, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-26T11-17-38/matrix.json", "duration_sec": 42.578327, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16075.448960997164, "mean_wall_clock_sec": 33.25269469898194, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-26T11-18-21/matrix.json", "duration_sec": 146.551389, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15615.472443401814, "mean_wall_clock_sec": 31.439707001671195, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-26T11-18-21/matrix.json", "duration_sec": 146.551389, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15767.563097178936, "mean_wall_clock_sec": 33.23396304342896, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-26T11-18-21/matrix.json", "duration_sec": 146.551389, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16384.749020449817, "mean_wall_clock_sec": 35.404665500856936, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-26T11-18-21/matrix.json", "duration_sec": 146.551389, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-27T14:52:24.181851+00:00", "build": { "amd_aorta_version": "0.2.2rc20260827", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33083741547", "head_sha": "bed2771d52ddc0d81cdf28ac176611815c2358c7", "wheel_file": "amd_aorta-0.2.2rc20260827-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 237.68535163253546, "mean_wall_clock_sec": 5.312802308239043, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-27T14-43-58/matrix.json", "duration_sec": 16.840055, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 35.86260904558003, "mean_wall_clock_sec": 11.787237565033138, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.0758277494460344, "decode_latency_ms": 0.9430897151750903, "tokens_per_sec": 4241.378031842258, "decoded_tokens": 496.0, "step_time_p50": 34.2475725337863, "step_time_p99": 41.13037567585707, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-27T14-44-15/matrix.json", "duration_sec": 23.496345, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1423.9123449660838, "mean_wall_clock_sec": 20.09583912137896, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.800723373889923, "step_time_p99": 3.340731058269739 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-27T14-44-40/matrix.json", "duration_sec": 34.415177, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1333.3767694421113, "mean_wall_clock_sec": 16.06265038996935, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 7.366158068180084, "step_time_p99": 22.19396622851491 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-27T14-45-14/matrix.json", "duration_sec": 30.205456, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2092.851400375366, "mean_wall_clock_sec": 17.732199531048536, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2092.851400375366, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-27T14-45-44/matrix.json", "duration_sec": 30.462349, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10837.659985758364, "mean_wall_clock_sec": 21.509499000385404, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-27T14-46-15/matrix.json", "duration_sec": 99.910074, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10505.804969929159, "mean_wall_clock_sec": 19.957638312131166, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-27T14-46-15/matrix.json", "duration_sec": 99.910074, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10594.634976238012, "mean_wall_clock_sec": 21.55916107352823, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-27T14-46-15/matrix.json", "duration_sec": 99.910074, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11658.65600015968, "mean_wall_clock_sec": 23.7992705963552, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-27T14-46-15/matrix.json", "duration_sec": 99.910074, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1467.266384512186, "mean_wall_clock_sec": 21.719660780392587, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.363702118396759, "step_time_p99": 35.56042704731226 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-27T14-47-55/matrix.json", "duration_sec": 41.382664, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1689.316704403609, "mean_wall_clock_sec": 22.45922456588596, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.635703146457672, "step_time_p99": 43.99185348302126 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-27T14-48-36/matrix.json", "duration_sec": 41.826664, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3318.1944847106934, "mean_wall_clock_sec": 28.819485569372773, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3318.1944847106934, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-27T14-49-18/matrix.json", "duration_sec": 42.482916, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16000.829330645502, "mean_wall_clock_sec": 32.752527214586735, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-27T14-50-00/matrix.json", "duration_sec": 144.49518, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15544.903999194503, "mean_wall_clock_sec": 31.100445928052068, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-27T14-50-00/matrix.json", "duration_sec": 144.49518, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15533.86761713773, "mean_wall_clock_sec": 32.6900955112651, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-27T14-50-00/matrix.json", "duration_sec": 144.49518, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16396.220626309514, "mean_wall_clock_sec": 35.09045338258147, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-27T14-50-00/matrix.json", "duration_sec": 144.49518, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-28T15:13:14.341124+00:00", "build": { "amd_aorta_version": "0.2.2rc20260828", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33183049651", "head_sha": "e61f455a130bf266c83730ac055cd208efe0af87", "wheel_file": "amd_aorta-0.2.2rc20260828-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 233.03094133734703, "mean_wall_clock_sec": 5.294548435136676, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-28T15-04-45/matrix.json", "duration_sec": 16.7368, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.752823766320944, "mean_wall_clock_sec": 12.217231031507254, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.10609270632267, "decode_latency_ms": 1.0209019340935253, "tokens_per_sec": 3918.104047429063, "decoded_tokens": 496.0, "step_time_p50": 37.942392751574516, "step_time_p99": 45.61169610358774, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-28T15-05-01/matrix.json", "duration_sec": 24.099166, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1450.7065927609801, "mean_wall_clock_sec": 20.362717929296196, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.2891143560409546, "step_time_p99": 38.91383100301027 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-28T15-05-27/matrix.json", "duration_sec": 34.462911, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1339.3266161438078, "mean_wall_clock_sec": 16.116171394474804, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.537106819450855, "step_time_p99": 8.560825195163488 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-28T15-06-01/matrix.json", "duration_sec": 30.306956, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2130.6937217712402, "mean_wall_clock_sec": 17.762714342214167, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2130.6937217712402, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-28T15-06-31/matrix.json", "duration_sec": 30.607689, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10711.310591548681, "mean_wall_clock_sec": 21.471342821605504, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-28T15-07-02/matrix.json", "duration_sec": 100.312675, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10542.535847052932, "mean_wall_clock_sec": 20.138327193446457, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-28T15-07-02/matrix.json", "duration_sec": 100.312675, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10613.461174070835, "mean_wall_clock_sec": 21.552714833058417, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-28T15-07-02/matrix.json", "duration_sec": 100.312675, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11592.277484945953, "mean_wall_clock_sec": 24.26537214498967, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-28T15-07-02/matrix.json", "duration_sec": 100.312675, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1487.2707372996956, "mean_wall_clock_sec": 22.36994189582765, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.2922734022140503, "step_time_p99": 38.07335216552019 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-28T15-08-42/matrix.json", "duration_sec": 41.621974, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1723.03794673644, "mean_wall_clock_sec": 22.520655094645917, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.629284471273422, "step_time_p99": 41.59582111984491 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-28T15-09-24/matrix.json", "duration_sec": 42.07726, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3396.658992767334, "mean_wall_clock_sec": 29.11248449422419, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3396.658992767334, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-28T15-10-06/matrix.json", "duration_sec": 42.527585, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16054.238038137555, "mean_wall_clock_sec": 32.85610403306782, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-28T15-10-48/matrix.json", "duration_sec": 146.533048, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16060.462861321867, "mean_wall_clock_sec": 31.804069097153842, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-28T15-10-48/matrix.json", "duration_sec": 146.533048, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16048.398580402136, "mean_wall_clock_sec": 32.82468974497169, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-28T15-10-48/matrix.json", "duration_sec": 146.533048, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 17075.737580657005, "mean_wall_clock_sec": 35.6772093558684, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-28T15-10-48/matrix.json", "duration_sec": 146.533048, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-29T11:17:34.959872+00:00", "build": { "amd_aorta_version": "0.2.2rc20260829", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33249416907", "head_sha": "e61f455a130bf266c83730ac055cd208efe0af87", "wheel_file": "amd_aorta-0.2.2rc20260829-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 237.26349417120218, "mean_wall_clock_sec": 5.286956084892154, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-29T11-09-06/matrix.json", "duration_sec": 16.786053, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 37.98878495581448, "mean_wall_clock_sec": 11.765718841925263, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1026870924979448, "decode_latency_ms": 0.9880527673709777, "tokens_per_sec": 4048.3667796844966, "decoded_tokens": 496.0, "step_time_p50": 36.04685561731458, "step_time_p99": 43.96283065900206, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-29T11-09-23/matrix.json", "duration_sec": 23.499759, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1447.1612942870706, "mean_wall_clock_sec": 20.044574710540473, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.324683427810669, "step_time_p99": 51.67655870318413 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-29T11-09-47/matrix.json", "duration_sec": 34.162807, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1349.4158224202693, "mean_wall_clock_sec": 16.072323152795434, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.866165436804295, "step_time_p99": 45.11398246511817 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-29T11-10-21/matrix.json", "duration_sec": 30.206747, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2102.9201984405518, "mean_wall_clock_sec": 17.827182991430163, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2102.9201984405518, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-29T11-10-51/matrix.json", "duration_sec": 30.756833, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10618.333172053099, "mean_wall_clock_sec": 21.62075128313154, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-29T11-11-22/matrix.json", "duration_sec": 100.151356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10583.63223541528, "mean_wall_clock_sec": 20.045707428827882, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-29T11-11-22/matrix.json", "duration_sec": 100.151356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10636.338961310685, "mean_wall_clock_sec": 21.498527728021145, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-29T11-11-22/matrix.json", "duration_sec": 100.151356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11568.282628431916, "mean_wall_clock_sec": 23.948053263127804, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-29T11-11-22/matrix.json", "duration_sec": 100.151356, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1493.1316890288144, "mean_wall_clock_sec": 21.923577594570816, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.446560025215149, "step_time_p99": 36.081848219037056 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-29T11-13-02/matrix.json", "duration_sec": 41.633574, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1716.932769631967, "mean_wall_clock_sec": 22.059908500872552, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.547773256897926, "step_time_p99": 43.174998834729195 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-29T11-13-44/matrix.json", "duration_sec": 42.178842, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3318.2244777679443, "mean_wall_clock_sec": 29.13971816096455, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3318.2244777679443, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-29T11-14-26/matrix.json", "duration_sec": 42.68491, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15507.551874034107, "mean_wall_clock_sec": 32.871820350177586, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-29T11-15-09/matrix.json", "duration_sec": 146.686742, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15690.232557244599, "mean_wall_clock_sec": 31.57164756115526, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-29T11-15-09/matrix.json", "duration_sec": 146.686742, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15793.611817061901, "mean_wall_clock_sec": 32.87221277691424, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-29T11-15-09/matrix.json", "duration_sec": 146.686742, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16648.148243315518, "mean_wall_clock_sec": 35.5941367438063, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-29T11-15-09/matrix.json", "duration_sec": 146.686742, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-30T11:18:55.728237+00:00", "build": { "amd_aorta_version": "0.2.2rc20260830", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33308201361", "head_sha": "c57a1e4b0720e483ffeba1d680111a717e6f3d6a", "wheel_file": "amd_aorta-0.2.2rc20260830-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 235.43531075119972, "mean_wall_clock_sec": 5.2978590885177255, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-30T11-10-29/matrix.json", "duration_sec": 16.738446, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.80873857997358, "mean_wall_clock_sec": 11.764142833650112, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.106842653825879, "decode_latency_ms": 1.0081015512227052, "tokens_per_sec": 3967.854225745893, "decoded_tokens": 496.0, "step_time_p50": 36.99840884655714, "step_time_p99": 44.68427014537155, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-30T11-10-46/matrix.json", "duration_sec": 23.500857, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1431.1091913841665, "mean_wall_clock_sec": 19.887236278504133, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.3814214617013931, "step_time_p99": 40.54005719721317 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-30T11-11-10/matrix.json", "duration_sec": 34.061269, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1325.4889689851552, "mean_wall_clock_sec": 15.935276614502072, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.688860245049, "step_time_p99": 8.418770786374807 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-30T11-11-44/matrix.json", "duration_sec": 29.858686, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2082.5549125671387, "mean_wall_clock_sec": 17.754254015162587, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2082.5549125671387, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-30T11-12-14/matrix.json", "duration_sec": 30.356224, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10717.983454465866, "mean_wall_clock_sec": 21.584149048663676, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-30T11-12-44/matrix.json", "duration_sec": 100.204853, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10654.396650381386, "mean_wall_clock_sec": 20.203136104159057, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-30T11-12-44/matrix.json", "duration_sec": 100.204853, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10514.917068183422, "mean_wall_clock_sec": 21.25752192735672, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-30T11-12-44/matrix.json", "duration_sec": 100.204853, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11956.77531324327, "mean_wall_clock_sec": 24.243634078651667, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-30T11-12-44/matrix.json", "duration_sec": 100.204853, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1469.2845491226763, "mean_wall_clock_sec": 21.587928999215364, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3795308768749237, "step_time_p99": 41.15910207852721 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-30T11-14-25/matrix.json", "duration_sec": 41.424256, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1735.8167606871575, "mean_wall_clock_sec": 22.32224070187658, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.675981916487217, "step_time_p99": 42.2248625010252 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-30T11-15-06/matrix.json", "duration_sec": 42.025146, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3284.8037242889404, "mean_wall_clock_sec": 29.336730958893895, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3284.8037242889404, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-30T11-15-48/matrix.json", "duration_sec": 42.834985, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15995.50854600966, "mean_wall_clock_sec": 32.75258920993656, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-30T11-16-31/matrix.json", "duration_sec": 145.474072, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15447.019052691758, "mean_wall_clock_sec": 30.998065315186977, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-30T11-16-31/matrix.json", "duration_sec": 145.474072, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15859.2065051198, "mean_wall_clock_sec": 33.0568927526474, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-30T11-16-31/matrix.json", "duration_sec": 145.474072, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16704.077397473156, "mean_wall_clock_sec": 35.39585995301604, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-30T11-16-31/matrix.json", "duration_sec": 145.474072, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-08-31T11:20:07.179562+00:00", "build": { "amd_aorta_version": "0.2.2rc20260831", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33385719574", "head_sha": "c57a1e4b0720e483ffeba1d680111a717e6f3d6a", "wheel_file": "amd_aorta-0.2.2rc20260831-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 237.62671276926994, "mean_wall_clock_sec": 5.27186796348542, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-08-31T11-11-39/matrix.json", "duration_sec": 16.6879, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 39.42392626777291, "mean_wall_clock_sec": 11.843320787884295, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1258912272751331, "decode_latency_ms": 1.025054386005767, "tokens_per_sec": 3902.2319738432843, "decoded_tokens": 496.0, "step_time_p50": 37.62901900336146, "step_time_p99": 45.13553746044635, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-08-31T11-11-56/matrix.json", "duration_sec": 23.645183, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1433.5662960074842, "mean_wall_clock_sec": 20.05060934368521, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.5118280425667763, "step_time_p99": 46.03284191340208 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-08-31T11-12-20/matrix.json", "duration_sec": 34.018183, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1349.214464891702, "mean_wall_clock_sec": 16.10563010815531, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.506472826004028, "step_time_p99": 8.507229592651129 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-08-31T11-12-55/matrix.json", "duration_sec": 30.556598, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2102.868413925171, "mean_wall_clock_sec": 17.771634845994413, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2102.868413925171, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-08-31T11-13-25/matrix.json", "duration_sec": 30.70713, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10476.37624386698, "mean_wall_clock_sec": 21.231289696879685, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-31T11-13-56/matrix.json", "duration_sec": 100.061059, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10572.569243609905, "mean_wall_clock_sec": 20.097117276862264, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-31T11-13-56/matrix.json", "duration_sec": 100.061059, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10680.922048166394, "mean_wall_clock_sec": 21.64473856240511, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-31T11-13-56/matrix.json", "duration_sec": 100.061059, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11457.863334566355, "mean_wall_clock_sec": 24.179750458337367, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-08-31T11-13-56/matrix.json", "duration_sec": 100.061059, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1490.0622782297432, "mean_wall_clock_sec": 22.671488201245666, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.3835513964295387, "step_time_p99": 37.535829078406096 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-08-31T11-15-36/matrix.json", "duration_sec": 42.235929, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1724.8141744639724, "mean_wall_clock_sec": 22.158266850747168, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.528004072606564, "step_time_p99": 49.040790759027004 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-08-31T11-16-18/matrix.json", "duration_sec": 41.836943, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3261.288833618164, "mean_wall_clock_sec": 28.764693945646286, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3261.288833618164, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-08-31T11-17-00/matrix.json", "duration_sec": 42.182927, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15682.784263975918, "mean_wall_clock_sec": 32.516369096934795, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-31T11-17-42/matrix.json", "duration_sec": 145.683904, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15780.109022744, "mean_wall_clock_sec": 31.744678515940905, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-31T11-17-42/matrix.json", "duration_sec": 145.683904, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15814.646208658814, "mean_wall_clock_sec": 32.91648401040584, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-31T11-17-42/matrix.json", "duration_sec": 145.683904, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16825.237959623337, "mean_wall_clock_sec": 35.14898759871721, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-08-31T11-17-42/matrix.json", "duration_sec": 145.683904, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-09-01T11:22:59.704763+00:00", "build": { "amd_aorta_version": "0.2.2rc20260901", "torch": "2.10.0+rocm7.2.4.git3d3aa833", "hip": "7.2.53211", "rocm": "7.2.4", "upstream_run_id": "33501157788", "head_sha": "d11092a603d9fe07f12046d144540be1fa80c264", "wheel_file": "amd_aorta-0.2.2rc20260901-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 236.4569939672947, "mean_wall_clock_sec": 5.279555697925389, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-09-01T11-14-31/matrix.json", "duration_sec": 16.686866, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 38.64844352938235, "mean_wall_clock_sec": 11.888707347214222, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.0977501515299082, "decode_latency_ms": 0.998216636118389, "tokens_per_sec": 4007.14619980106, "decoded_tokens": 496.0, "step_time_p50": 36.95025760680437, "step_time_p99": 44.24825306050479, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-09-01T11-14-48/matrix.json", "duration_sec": 23.597286, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1456.6396696027368, "mean_wall_clock_sec": 20.102265728637576, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.9368743896484375, "step_time_p50": 1.4612497761845589, "step_time_p99": 44.82549615204334 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-09-01T11-15-13/matrix.json", "duration_sec": 34.315, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1341.9804638251662, "mean_wall_clock_sec": 16.106601225212216, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.3885498046875, "step_time_p50": 6.473190151154995, "step_time_p99": 29.889034386724234 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-09-01T11-15-47/matrix.json", "duration_sec": 30.459759, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 2110.570812225342, "mean_wall_clock_sec": 17.86335892789066, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 2110.570812225342, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-09-01T11-16-17/matrix.json", "duration_sec": 30.70889, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10691.256534308195, "mean_wall_clock_sec": 21.657865082845092, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-01T11-16-48/matrix.json", "duration_sec": 100.552395, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10828.572678379714, "mean_wall_clock_sec": 19.962925137020648, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-01T11-16-48/matrix.json", "duration_sec": 100.552395, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 10662.254142574966, "mean_wall_clock_sec": 21.558517469093204, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-01T11-16-48/matrix.json", "duration_sec": 100.552395, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 11560.082483105361, "mean_wall_clock_sec": 24.303495631553233, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-01T11-16-48/matrix.json", "duration_sec": 100.552395, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1463.849373627454, "mean_wall_clock_sec": 21.91223955154419, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936972618103027, "step_time_p50": 1.2322254478931427, "step_time_p99": 34.912614449858665 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-09-01T11-18-29/matrix.json", "duration_sec": 41.924416, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1714.5502131897956, "mean_wall_clock_sec": 22.214799246750772, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.35585021972656, "step_time_p50": 6.752843037247658, "step_time_p99": 34.08669367432594 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-09-01T11-19-11/matrix.json", "duration_sec": 41.731105, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 3333.7934494018555, "mean_wall_clock_sec": 28.900008539669216, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 3333.7934494018555, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-09-01T11-19-52/matrix.json", "duration_sec": 41.875615, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15512.476848438382, "mean_wall_clock_sec": 33.15654428768903, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-01T11-20-34/matrix.json", "duration_sec": 146.184835, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15559.712324291468, "mean_wall_clock_sec": 31.035339736379683, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-01T11-20-34/matrix.json", "duration_sec": 146.184835, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 15615.694474428892, "mean_wall_clock_sec": 32.860103480517864, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-01T11-20-34/matrix.json", "duration_sec": 146.184835, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 16935.27401331812, "mean_wall_clock_sec": 35.72388849593699, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-01T11-20-34/matrix.json", "duration_sec": 146.184835, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-09-02T11:18:10.980180+00:00", "build": { "amd_aorta_version": "0.3.1rc20260902", "torch": "2.13.0+rocm10.0.0", "hip": "7.15.26333", "rocm": "10.0.0", "upstream_run_id": "33623244658", "head_sha": "30dcc055cba8f7f335169014f9af090a83ba00bd", "wheel_file": "amd_aorta-0.3.1rc20260902-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 205.10198920965195, "mean_wall_clock_sec": 3.584490125067532, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-09-02T11-12-08/matrix.json", "duration_sec": 11.679501, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 45.83283932879567, "mean_wall_clock_sec": 10.457494990900159, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.2187492102384567, "decode_latency_ms": 1.2380406185383759, "tokens_per_sec": 3230.911765013315, "decoded_tokens": 496.0, "step_time_p50": 44.03874417766929, "step_time_p99": 51.26236298121512, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-09-02T11-12-20/matrix.json", "duration_sec": 19.089647, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1393.939848523587, "mean_wall_clock_sec": 11.785226366482675, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.936877250671387, "step_time_p50": 1.14450603723526, "step_time_p99": 55.949342641979456 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-09-02T11-12-40/matrix.json", "duration_sec": 23.044965, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1316.283417167142, "mean_wall_clock_sec": 11.58822969906032, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.41094970703125, "step_time_p50": 5.500185303390026, "step_time_p99": 35.27277823537588 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-09-02T11-13-03/matrix.json", "duration_sec": 22.845523, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1517.3772811889648, "mean_wall_clock_sec": 13.176710119470954, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1517.3772811889648, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-09-02T11-13-26/matrix.json", "duration_sec": 22.74605, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7736.535225994885, "mean_wall_clock_sec": 17.50717061199248, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-02T11-13-49/matrix.json", "duration_sec": 80.379356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7694.487731903791, "mean_wall_clock_sec": 16.00576878990978, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-02T11-13-49/matrix.json", "duration_sec": 80.379356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7843.435263261199, "mean_wall_clock_sec": 17.695268664509058, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-02T11-13-49/matrix.json", "duration_sec": 80.379356, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8122.953820973635, "mean_wall_clock_sec": 19.342709354124963, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-02T11-13-49/matrix.json", "duration_sec": 80.379356, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1367.766277398914, "mean_wall_clock_sec": 12.668337318114936, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936975479125977, "step_time_p50": 1.2100841850042343, "step_time_p99": 56.29063807427883 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-09-02T11-15-09/matrix.json", "duration_sec": 27.552657, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1383.5159637965262, "mean_wall_clock_sec": 12.631039250642061, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.3743896484375, "step_time_p50": 5.906447768211365, "step_time_p99": 63.870966248214245 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-09-02T11-15-37/matrix.json", "duration_sec": 27.403138, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1738.4585857391357, "mean_wall_clock_sec": 17.678570765070617, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1738.4585857391357, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-09-02T11-16-04/matrix.json", "duration_sec": 27.955368, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8840.177766047418, "mean_wall_clock_sec": 22.11822633445263, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-02T11-16-32/matrix.json", "duration_sec": 99.560817, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8762.53956835717, "mean_wall_clock_sec": 20.390193484723568, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-02T11-16-32/matrix.json", "duration_sec": 99.560817, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8771.664313971996, "mean_wall_clock_sec": 22.30983843933791, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-02T11-16-32/matrix.json", "duration_sec": 99.560817, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9236.95612885058, "mean_wall_clock_sec": 24.230436332523823, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-02T11-16-32/matrix.json", "duration_sec": 99.560817, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-09-03T15:58:15.804087+00:00", "build": { "amd_aorta_version": "0.3.1rc20260903", "torch": "2.13.0+rocm10.0.0", "hip": "7.15.26333", "rocm": "10.0.0", "upstream_run_id": "33770514024", "head_sha": "00c4a4609a45ea90503a4329c882ea1c416871c3", "wheel_file": "amd_aorta-0.3.1rc20260903-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 194.91543993353844, "mean_wall_clock_sec": 3.4853040548041463, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-09-03T15-52-15/matrix.json", "duration_sec": 11.427726, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 43.08691783808172, "mean_wall_clock_sec": 10.258311201818287, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.125152688473463, "decode_latency_ms": 1.154745643537852, "tokens_per_sec": 3463.9663049474684, "decoded_tokens": 496.0, "step_time_p50": 41.243262123316526, "step_time_p99": 48.741095708683126, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-09-03T15-52-27/matrix.json", "duration_sec": 18.590283, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1385.4475219268352, "mean_wall_clock_sec": 11.890781846828759, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.936877250671387, "step_time_p50": 0.8973916992545128, "step_time_p99": 2.5848207995295525 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-09-03T15-52-47/matrix.json", "duration_sec": 23.044073, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1334.754404379055, "mean_wall_clock_sec": 11.82110760267824, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.41094970703125, "step_time_p50": 6.0476278886199, "step_time_p99": 35.13858010992408 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-09-03T15-53-10/matrix.json", "duration_sec": 22.993773, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1595.0532913208008, "mean_wall_clock_sec": 13.324308324605227, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1595.0532913208008, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-09-03T15-53-32/matrix.json", "duration_sec": 22.696308, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7749.101537279785, "mean_wall_clock_sec": 17.32785391435027, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-03T15-53-55/matrix.json", "duration_sec": 79.765642, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7514.7407706826925, "mean_wall_clock_sec": 15.42016976326704, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-03T15-53-55/matrix.json", "duration_sec": 79.765642, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7810.342678800225, "mean_wall_clock_sec": 17.68350158073008, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-03T15-53-55/matrix.json", "duration_sec": 79.765642, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8183.645701035857, "mean_wall_clock_sec": 19.42276345938444, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-03T15-53-55/matrix.json", "duration_sec": 79.765642, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1344.8200300335884, "mean_wall_clock_sec": 12.96160705666989, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936975479125977, "step_time_p50": 1.3252338394522667, "step_time_p99": 61.52577659115195 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-09-03T15-55-15/matrix.json", "duration_sec": 27.402301, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1376.2192348949611, "mean_wall_clock_sec": 12.85022970661521, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.3743896484375, "step_time_p50": 5.701015703380108, "step_time_p99": 59.7945361956954 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-09-03T15-55-42/matrix.json", "duration_sec": 28.053027, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1828.354024887085, "mean_wall_clock_sec": 17.983784468844533, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1828.354024887085, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-09-03T15-56-10/matrix.json", "duration_sec": 27.951704, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8764.764130115509, "mean_wall_clock_sec": 21.842568710446358, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-03T15-56-38/matrix.json", "duration_sec": 98.050438, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8600.711174309254, "mean_wall_clock_sec": 19.941322376951575, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-03T15-56-38/matrix.json", "duration_sec": 98.050438, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8716.886210255325, "mean_wall_clock_sec": 21.891309496946633, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-03T15-56-38/matrix.json", "duration_sec": 98.050438, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9102.779981680214, "mean_wall_clock_sec": 24.03590943850577, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-03T15-56-38/matrix.json", "duration_sec": 98.050438, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-09-04T12:07:27.268542+00:00", "build": { "amd_aorta_version": "0.3.1rc20260904", "torch": "2.13.0+rocm10.0.0", "hip": "7.15.26333", "rocm": "10.0.0", "upstream_run_id": "33866672651", "head_sha": "4b4553ef14af08e7ee2d68a888b68cd6a88803f7", "wheel_file": "amd_aorta-0.3.1rc20260904-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 197.1269454807043, "mean_wall_clock_sec": 3.488311585970223, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-09-04T12-01-27/matrix.json", "duration_sec": 11.378181, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 46.52944556437433, "mean_wall_clock_sec": 10.306078616529703, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.2305446434766054, "decode_latency_ms": 1.253211660490882, "tokens_per_sec": 3191.7992196411606, "decoded_tokens": 496.0, "step_time_p50": 44.70795299857855, "step_time_p99": 51.97530409321189, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-09-04T12-01-38/matrix.json", "duration_sec": 18.98768, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1392.7845461294055, "mean_wall_clock_sec": 11.813615767285228, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.936877250671387, "step_time_p50": 1.300223171710968, "step_time_p99": 2.4695648811757565 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-09-04T12-01-58/matrix.json", "duration_sec": 22.692251, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1315.759169869125, "mean_wall_clock_sec": 11.578911766409874, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.41094970703125, "step_time_p50": 6.064905785024166, "step_time_p99": 73.07050261646509 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-09-04T12-02-21/matrix.json", "duration_sec": 22.594728, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1498.401927947998, "mean_wall_clock_sec": 13.05800417624414, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1498.401927947998, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-09-04T12-02-43/matrix.json", "duration_sec": 22.642641, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7754.960438236594, "mean_wall_clock_sec": 17.033519191667438, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-04T12-03-06/matrix.json", "duration_sec": 79.464545, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7825.679431669414, "mean_wall_clock_sec": 15.449163551442325, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-04T12-03-06/matrix.json", "duration_sec": 79.464545, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8188.803247176111, "mean_wall_clock_sec": 18.029674901627004, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-04T12-03-06/matrix.json", "duration_sec": 79.464545, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8019.5531994104385, "mean_wall_clock_sec": 19.048223710618913, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-04T12-03-06/matrix.json", "duration_sec": 79.464545, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1345.827337820083, "mean_wall_clock_sec": 12.635099803097546, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936975479125977, "step_time_p50": 1.222384162247181, "step_time_p99": 43.737591318786144 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-09-04T12-04-25/matrix.json", "duration_sec": 27.451018, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1385.7448492199183, "mean_wall_clock_sec": 13.311718198470771, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.3743896484375, "step_time_p50": 6.177191622555256, "step_time_p99": 66.84651426970959 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-09-04T12-04-53/matrix.json", "duration_sec": 27.501668, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1758.1916809082031, "mean_wall_clock_sec": 18.001809938810766, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1758.1916809082031, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-09-04T12-05-21/matrix.json", "duration_sec": 28.304009, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9545.81649787724, "mean_wall_clock_sec": 22.307418304495513, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-04T12-05-49/matrix.json", "duration_sec": 99.057557, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8820.134059526026, "mean_wall_clock_sec": 20.416445516049862, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-04T12-05-49/matrix.json", "duration_sec": 99.057557, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8859.766811132431, "mean_wall_clock_sec": 22.19096360448748, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-04T12-05-49/matrix.json", "duration_sec": 99.057557, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9440.915632992983, "mean_wall_clock_sec": 24.36372254882008, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-04T12-05-49/matrix.json", "duration_sec": 99.057557, "error": null } ] }, { "schema_version": 1, "generated_at": "2026-09-05T11:17:02.281130+00:00", "build": { "amd_aorta_version": "0.3.1rc20260905", "torch": "2.13.0+rocm10.0.0", "hip": "7.15.26333", "rocm": "10.0.0", "upstream_run_id": "33962529149", "head_sha": "4b4553ef14af08e7ee2d68a888b68cd6a88803f7", "wheel_file": "amd_aorta-0.3.1rc20260905-py3-none-any.whl", "lane": "gate", "base_image": null }, "summary": { "total": 16, "pass": 16, "fail": 0, "record": 0, "skip": 0 }, "entries": [ { "entry": "gpu_smoke", "recipe": "recipes/ci/gpu-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 198.89609329402447, "mean_wall_clock_sec": 3.5157844461500645, "step_time_source": "elapsed_per_iter", "summary": { "sum": 8.0, "expected": 8.0, "n": 8.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/gpu_smoke/CI-GPU-SMOKE/gpu_smoke/2026-09-05T11-11-03/matrix.json", "duration_sec": 11.678686, "error": null }, { "entry": "inference_offline", "recipe": "recipes/inference/example-inference-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 42.53852320834994, "mean_wall_clock_sec": 10.420240699313581, "step_time_source": "per_step", "summary": { "parameter_count": 33170432.0, "batch_size": 4.0, "prompt_len": 128.0, "generate_tokens": 32.0, "prefill_latency_ms": 1.1212588287889957, "decode_latency_ms": 1.1428831024996695, "tokens_per_sec": 3499.9205004005703, "decoded_tokens": 496.0, "step_time_p50": 40.75635364279151, "step_time_p99": 47.888710154220455, "logits_checksum": 33904201.0 } }, "deltas": { "metrics": { "logits_checksum": { "observed": 33904201.0, "policy": "equal", "value": 33904201.0 } } }, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/inference_offline/EXAMPLE-INFERENCE-SMOKE/inference/2026-09-05T11-11-14/matrix.json", "duration_sec": 18.788014, "error": null }, { "entry": "training_ddp", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1356.6112278494984, "mean_wall_clock_sec": 11.744176439009607, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 656896.0, "final_loss": 6.936877250671387, "step_time_p50": 1.1385362595319748, "step_time_p99": 46.43335580825806 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp/EXAMPLE-TRAINING-DDP/training/2026-09-05T11-11-34/matrix.json", "duration_sec": 22.893147, "error": null }, { "entry": "training_fsdp", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1304.4884360861033, "mean_wall_clock_sec": 11.78439718671143, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 2.0, "parameter_count": 2361856.0, "final_loss": 254.41094970703125, "step_time_p50": 5.3746672347188, "step_time_p99": 7.403842620551586 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp/EXAMPLE-TRAINING-FSDP/training/2026-09-05T11-11-57/matrix.json", "duration_sec": 22.642792, "error": null }, { "entry": "race", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1497.8277206420898, "mean_wall_clock_sec": 13.085150383412838, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1497.8277206420898, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 2.0, "local_world_size": 2.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race/RACE-SMOKE/race/2026-09-05T11-12-20/matrix.json", "duration_sec": 22.693035, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8098.414341919124, "mean_wall_clock_sec": 17.223711256869137, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-05T11-12-42/matrix.json", "duration_sec": 79.217875, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7784.283955581486, "mean_wall_clock_sec": 15.709117466583848, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-05T11-12-42/matrix.json", "duration_sec": 79.217875, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 7973.096179775894, "mean_wall_clock_sec": 17.133240179158747, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-05T11-12-42/matrix.json", "duration_sec": 79.217875, "error": null }, { "entry": "llm_determinism", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8567.701815627515, "mean_wall_clock_sec": 19.414809589274228, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 2.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism/LLM-DET-001/llm_determinism/2026-09-05T11-12-42/matrix.json", "duration_sec": 79.217875, "error": null }, { "entry": "training_ddp_8gpu", "recipe": "recipes/training/example-training-ddp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1362.7359247766435, "mean_wall_clock_sec": 12.938980787061155, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 656896.0, "final_loss": 6.936975479125977, "step_time_p50": 1.2575546279549599, "step_time_p99": 58.03547140210867 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_ddp_8gpu/EXAMPLE-TRAINING-DDP/training/2026-09-05T11-14-02/matrix.json", "duration_sec": 27.600883, "error": null }, { "entry": "training_fsdp_8gpu", "recipe": "recipes/training/example-training-fsdp-smoke.yaml", "cell": "baseline-local", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1349.0914076101035, "mean_wall_clock_sec": 12.892769887112081, "step_time_source": "per_step", "summary": { "rank": 0.0, "world_size": 8.0, "parameter_count": 2361856.0, "final_loss": 254.3743896484375, "step_time_p50": 5.691795609891415, "step_time_p99": 59.068138655275106 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/training_fsdp_8gpu/EXAMPLE-TRAINING-FSDP/training/2026-09-05T11-14-29/matrix.json", "duration_sec": 27.551461, "error": null }, { "entry": "race_8gpu", "recipe": "recipes/race/race_smoke.yaml", "cell": "smoke", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 1780.3287029266357, "mean_wall_clock_sec": 17.67631482705474, "step_time_source": "elapsed_per_iter", "summary": { "avg_step_time_ms": 1780.3287029266357, "layers_verified": 15.0, "layer_checksum_mismatches": 0.0, "eff_num_heads": 4.0, "eff_ffn_size": 2048.0, "eff_seq_len": 512.0, "eff_batch_size": 1.0, "effective_h2d_tensor_size": 1000000.0, "declared_h2d_tensor_size": 1000000.0, "corruption_details_omitted": 0.0, "rank": 0.0, "world_size": 8.0, "local_world_size": 8.0, "node_count": 1.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/race_8gpu/RACE-SMOKE/race/2026-09-05T11-14-57/matrix.json", "duration_sec": 27.452322, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "baseline-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9020.401783287525, "mean_wall_clock_sec": 22.166775257326663, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-05T11-15-24/matrix.json", "duration_sec": 98.653922, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8692.848742939532, "mean_wall_clock_sec": 20.357651338912547, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-05T11-15-24/matrix.json", "duration_sec": 98.653922, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "tf32_off-bf16-24L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 8775.68987943232, "mean_wall_clock_sec": 21.910577720962465, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 24.0, "num_experts": 1.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-05T11-15-24/matrix.json", "duration_sec": 98.653922, "error": null }, { "entry": "llm_determinism_8gpu", "recipe": "recipes/llm-determinism/example-llm-determinism.yaml", "cell": "moe4-bf16-12L", "verdict": "pass", "reasons": [], "metrics": { "mean_step_time_ms": 9230.694468133152, "mean_wall_clock_sec": 24.061431881040335, "step_time_source": "elapsed_per_iter", "summary": { "rank": 0.0, "world_size": 8.0, "num_layers": 12.0, "num_experts": 4.0, "ranks_with_divergence": 0.0, "corruption_details_omitted": 0.0 } }, "deltas": {}, "passed_count": 1, "failed_count": 0, "error_count": 0, "trials": 1, "matrix_path": ".nightly-eval/llm_determinism_8gpu/LLM-DET-001/llm_determinism/2026-09-05T11-15-24/matrix.json", "duration_sec": 98.653922, "error": null } ] } ]