{
  "recorded_date": "2026-10-27",
  "status": "completed",
  "scope": "Completed E18, E19 pilot, E19-funded and E20 recorded results. Retained aggregate reports and benchmark metadata; E20 provenance limits are explicit.",
  "experiments": {
    "sudoku-e18": {
      "config": "configs/sudoku-e18.json",
      "generator_sha256": "93794833b6d5c7b2364ec72863a4b9f78ec6fa2e6cfbf4e058175c97921b53f0",
      "training": {
        "steps": 128000,
        "best_step": 128000,
        "train_seconds": 23366.69
      },
      "batch_size": 64,
      "sequence_presentations": 8192000,
      "runtime": {
        "mlx": "0.32.3",
        "device": "Device(gpu, 0)",
        "metal": false,
        "cuda": true,
        "machine": "x86_64",
        "macos": "",
        "python": "3.12.15",
        "chip": "NVIDIA GeForce RTX 3080",
        "memory_gb": 0.0
      },
      "parameters": {
        "codec": 13915200,
        "generator": 6440128
      },
      "codec_validation": {
        "char_acc_by_levels_used": [
          0.1573,
          0.157,
          0.1578,
          0.1638,
          0.1635,
          0.2002,
          0.2593,
          0.3349,
          0.4487,
          0.5671,
          0.7239,
          1.0
        ],
        "seq_exact_by_levels_used": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          1.0
        ],
        "codebook_utilization": [
          1.0,
          0.9375,
          0.6875,
          0.625,
          0.5,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0
        ],
        "code_perplexity": [
          15.93,
          14.68,
          10.79,
          9.83,
          7.07,
          15.77,
          15.49,
          15.83,
          15.92,
          15.56,
          15.45,
          14.53
        ]
      },
      "generation_temperature": 0.0,
      "quality": {
        "val": {
          "solved": 0.558,
          "valid_grid": 0.558,
          "respects_givens": 0.992,
          "cell_acc": 0.8889,
          "blank_cell_acc": 0.8122,
          "n": 1000.0
        },
        "upstream-easy": {
          "solved": 0.8425,
          "valid_grid": 0.843,
          "respects_givens": 0.998,
          "cell_acc": 0.9873,
          "blank_cell_acc": 0.975,
          "n": 2000.0
        },
        "upstream-medium": {
          "solved": 0.491,
          "valid_grid": 0.491,
          "respects_givens": 0.9965,
          "cell_acc": 0.9467,
          "blank_cell_acc": 0.9062,
          "n": 2000.0
        },
        "upstream-hard": {
          "solved": 0.1055,
          "valid_grid": 0.106,
          "respects_givens": 0.9855,
          "cell_acc": 0.8531,
          "blank_cell_acc": 0.767,
          "n": 2000.0
        }
      },
      "benchmark": {
        "runtime": {
          "mlx": "0.32.3",
          "device": "Device(gpu, 0)",
          "metal": false,
          "cuda": true,
          "machine": "x86_64",
          "macos": "",
          "python": "3.12.15",
          "chip": "NVIDIA GeForce RTX 3080",
          "memory_gb": 0.0
        },
        "seq_len": 81,
        "levels": [
          1,
          2,
          4,
          6,
          9,
          13,
          18,
          26,
          37,
          50,
          67,
          81
        ],
        "rows": [
          {
            "batch": 1,
            "first_call_ms": 320.92,
            "p50_ms": 15.952,
            "p95_ms": 17.269,
            "mean_ms": 16.264,
            "stdev_ms": 0.595,
            "sequences_per_s": 62.7,
            "chars_per_s": 5078.0,
            "peak_memory_mb": 141.4,
            "model_forwards": 12
          },
          {
            "batch": 16,
            "first_call_ms": 44.89,
            "p50_ms": 44.537,
            "p95_ms": 44.659,
            "mean_ms": 44.484,
            "stdev_ms": 0.172,
            "sequences_per_s": 359.3,
            "chars_per_s": 29099.0,
            "peak_memory_mb": 552.4,
            "model_forwards": 12
          },
          {
            "batch": 64,
            "first_call_ms": 142.48,
            "p50_ms": 153.8,
            "p95_ms": 154.312,
            "mean_ms": 153.68,
            "stdev_ms": 0.459,
            "sequences_per_s": 416.1,
            "chars_per_s": 33706.0,
            "peak_memory_mb": 1911.2,
            "model_forwards": 12
          },
          {
            "batch": 256,
            "first_call_ms": 512.91,
            "p50_ms": 524.874,
            "p95_ms": 525.89,
            "mean_ms": 522.799,
            "stdev_ms": 4.529,
            "sequences_per_s": 487.7,
            "chars_per_s": 39507.0,
            "peak_memory_mb": 2981.0,
            "model_forwards": 12
          }
        ]
      },
      "wilson95_solved": {
        "val": [
          0.527056,
          0.5885
        ],
        "upstream-easy": [
          0.825881,
          0.857806
        ],
        "upstream-medium": [
          0.469129,
          0.512906
        ],
        "upstream-hard": [
          0.092785,
          0.119728
        ]
      }
    },
    "sudoku-e19-budget": {
      "config": "configs/sudoku-e19-budget.json",
      "generator_sha256": "34ce3edeec7383fc8984af50ed2bb3ea2c45889215608cf509a195639021b638",
      "training": {
        "steps": 23895,
        "best_step": 23895,
        "train_seconds": 10225.84
      },
      "batch_size": 128,
      "sequence_presentations": 3058560,
      "runtime": {
        "mlx": "0.32.3",
        "device": "Device(gpu, 0)",
        "metal": false,
        "cuda": true,
        "machine": "x86_64",
        "macos": "",
        "python": "3.12.3",
        "chip": "NVIDIA A40",
        "memory_gb": 0.0
      },
      "parameters": {
        "codec": 13915200,
        "generator": 6440128
      },
      "codec_validation": {
        "char_acc_by_levels_used": [
          0.1573,
          0.157,
          0.1578,
          0.1638,
          0.1635,
          0.2002,
          0.2593,
          0.3349,
          0.4487,
          0.5671,
          0.7239,
          1.0
        ],
        "seq_exact_by_levels_used": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          1.0
        ],
        "codebook_utilization": [
          1.0,
          0.9375,
          0.6875,
          0.625,
          0.5,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0
        ],
        "code_perplexity": [
          15.93,
          14.68,
          10.79,
          9.83,
          7.07,
          15.77,
          15.49,
          15.83,
          15.92,
          15.56,
          15.45,
          14.53
        ]
      },
      "generation_temperature": 0.0,
      "quality": {
        "val": {
          "solved": 0.624,
          "valid_grid": 0.624,
          "respects_givens": 0.987,
          "cell_acc": 0.8937,
          "blank_cell_acc": 0.821,
          "n": 1000.0
        },
        "upstream-easy": {
          "solved": 0.902,
          "valid_grid": 0.902,
          "respects_givens": 0.999,
          "cell_acc": 0.9922,
          "blank_cell_acc": 0.9847,
          "n": 2000.0
        },
        "upstream-medium": {
          "solved": 0.5975,
          "valid_grid": 0.6005,
          "respects_givens": 0.9905,
          "cell_acc": 0.9589,
          "blank_cell_acc": 0.9278,
          "n": 2000.0
        },
        "upstream-hard": {
          "solved": 0.1655,
          "valid_grid": 0.1685,
          "respects_givens": 0.964,
          "cell_acc": 0.8647,
          "blank_cell_acc": 0.7859,
          "n": 2000.0
        }
      },
      "benchmark": {
        "runtime": {
          "mlx": "0.32.3",
          "device": "Device(gpu, 0)",
          "metal": false,
          "cuda": true,
          "machine": "x86_64",
          "macos": "",
          "python": "3.12.3",
          "chip": "NVIDIA A40",
          "memory_gb": 0.0
        },
        "seq_len": 81,
        "levels": [
          1,
          2,
          4,
          6,
          9,
          13,
          18,
          26,
          37,
          50,
          67,
          81
        ],
        "temperature": 1.0,
        "warmup": 3,
        "iterations": 20,
        "sampler_compiled": false,
        "synchronization": "numpy_host_copy",
        "rows": [
          {
            "batch": 1,
            "first_call_ms": 517.73,
            "p50_ms": 37.415,
            "p95_ms": 39.503,
            "mean_ms": 37.565,
            "stdev_ms": 1.252,
            "sequences_per_s": 26.7,
            "chars_per_s": 2165.0,
            "peak_memory_mb": 127.2,
            "model_forwards": 12
          },
          {
            "batch": 16,
            "first_call_ms": 89.37,
            "p50_ms": 51.42,
            "p95_ms": 52.599,
            "mean_ms": 51.445,
            "stdev_ms": 1.268,
            "sequences_per_s": 311.2,
            "chars_per_s": 25204.0,
            "peak_memory_mb": 552.5,
            "model_forwards": 12
          },
          {
            "batch": 64,
            "first_call_ms": 158.78,
            "p50_ms": 174.178,
            "p95_ms": 181.422,
            "mean_ms": 174.828,
            "stdev_ms": 2.492,
            "sequences_per_s": 367.4,
            "chars_per_s": 29763.0,
            "peak_memory_mb": 1911.2,
            "model_forwards": 12
          },
          {
            "batch": 256,
            "first_call_ms": 566.19,
            "p50_ms": 582.918,
            "p95_ms": 587.767,
            "mean_ms": 582.703,
            "stdev_ms": 3.494,
            "sequences_per_s": 439.2,
            "chars_per_s": 35573.0,
            "peak_memory_mb": 2981.0,
            "model_forwards": 12
          }
        ]
      },
      "wilson95_solved": {
        "val": [
          0.593558,
          0.653493
        ],
        "upstream-easy": [
          0.888189,
          0.91427
        ],
        "upstream-medium": [
          0.57584,
          0.618786
        ],
        "upstream-hard": [
          0.149857,
          0.182425
        ]
      }
    },
    "sudoku-e19-funded": {
      "config": "configs/sudoku-e19-funded.json",
      "generator_sha256": "843a8929c43ae8a976e7a71402d18d9a5ce7172ebfc3b3870cfe74fc3c8293b9",
      "training": {
        "steps": 128000,
        "best_step": 127000,
        "train_seconds": 54419.81
      },
      "batch_size": 128,
      "sequence_presentations": 16384000,
      "runtime": {
        "mlx": "0.32.3",
        "device": "Device(gpu, 0)",
        "metal": false,
        "cuda": true,
        "machine": "x86_64",
        "macos": "",
        "python": "3.12.3",
        "chip": "NVIDIA A40",
        "memory_gb": 0.0
      },
      "parameters": {
        "codec": 13915200,
        "generator": 6440128
      },
      "codec_validation": {
        "char_acc_by_levels_used": [
          0.1573,
          0.157,
          0.1578,
          0.1638,
          0.1635,
          0.2002,
          0.2593,
          0.3349,
          0.4487,
          0.5671,
          0.7239,
          1.0
        ],
        "seq_exact_by_levels_used": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          1.0
        ],
        "codebook_utilization": [
          1.0,
          0.9375,
          0.6875,
          0.625,
          0.5,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0
        ],
        "code_perplexity": [
          15.93,
          14.68,
          10.79,
          9.83,
          7.07,
          15.77,
          15.49,
          15.83,
          15.92,
          15.56,
          15.45,
          14.53
        ]
      },
      "generation_temperature": 0.0,
      "quality": {
        "val": {
          "solved": 0.709,
          "valid_grid": 0.709,
          "respects_givens": 0.998,
          "cell_acc": 0.9014,
          "blank_cell_acc": 0.8341,
          "n": 1000.0
        },
        "upstream-easy": {
          "solved": 0.981,
          "valid_grid": 0.981,
          "respects_givens": 1.0,
          "cell_acc": 0.9984,
          "blank_cell_acc": 0.9969,
          "n": 2000.0
        },
        "upstream-medium": {
          "solved": 0.834,
          "valid_grid": 0.834,
          "respects_givens": 1.0,
          "cell_acc": 0.9836,
          "blank_cell_acc": 0.9712,
          "n": 2000.0
        },
        "upstream-hard": {
          "solved": 0.403,
          "valid_grid": 0.404,
          "respects_givens": 0.996,
          "cell_acc": 0.9181,
          "blank_cell_acc": 0.8699,
          "n": 2000.0
        }
      },
      "benchmark": {
        "runtime": {
          "mlx": "0.32.3",
          "device": "Device(gpu, 0)",
          "metal": false,
          "cuda": true,
          "machine": "x86_64",
          "macos": "",
          "python": "3.12.3",
          "chip": "NVIDIA A40",
          "memory_gb": 0.0
        },
        "seq_len": 81,
        "levels": [
          1,
          2,
          4,
          6,
          9,
          13,
          18,
          26,
          37,
          50,
          67,
          81
        ],
        "temperature": 1.0,
        "warmup": 3,
        "iterations": 20,
        "sampler_compiled": false,
        "synchronization": "numpy_host_copy",
        "rows": [
          {
            "batch": 1,
            "first_call_ms": 516.39,
            "p50_ms": 26.477,
            "p95_ms": 26.69,
            "mean_ms": 26.494,
            "stdev_ms": 0.135,
            "sequences_per_s": 37.8,
            "chars_per_s": 3059.0,
            "peak_memory_mb": 130.2,
            "model_forwards": 12
          },
          {
            "batch": 16,
            "first_call_ms": 52.6,
            "p50_ms": 47.905,
            "p95_ms": 48.308,
            "mean_ms": 47.856,
            "stdev_ms": 0.393,
            "sequences_per_s": 334.0,
            "chars_per_s": 27054.0,
            "peak_memory_mb": 552.4,
            "model_forwards": 12
          },
          {
            "batch": 64,
            "first_call_ms": 155.76,
            "p50_ms": 174.279,
            "p95_ms": 174.552,
            "mean_ms": 174.257,
            "stdev_ms": 0.276,
            "sequences_per_s": 367.2,
            "chars_per_s": 29745.0,
            "peak_memory_mb": 1911.2,
            "model_forwards": 12
          },
          {
            "batch": 256,
            "first_call_ms": 624.38,
            "p50_ms": 583.67,
            "p95_ms": 584.715,
            "mean_ms": 583.761,
            "stdev_ms": 0.519,
            "sequences_per_s": 438.6,
            "chars_per_s": 35527.0,
            "peak_memory_mb": 2981.0,
            "model_forwards": 12
          }
        ]
      },
      "wilson95_solved": {
        "val": [
          0.68009,
          0.73631
        ],
        "upstream-easy": [
          0.97403,
          0.986126
        ],
        "upstream-medium": [
          0.817056,
          0.849663
        ],
        "upstream-hard": [
          0.381709,
          0.424663
        ]
      },
      "training_seed": 0
    },
    "sudoku-e20-seed1": {
      "config": "configs/sudoku-e20-seed1.json",
      "source_commit": "4ad3646189197fe1ca76e45308848793fefe9a3f",
      "source_identity_status": "Controller-declared commit; no remote source digest inventory retained.",
      "generator_sha256": "b6ddadb7278e7f1c5542eecef676840f39cb829e802d189b65f2dadd645c2b3d",
      "evaluation_sha256": "4626617cc962794c34917fe86be15160f387d8c049a85273aad396cc3f2102e4",
      "training": {
        "steps": 128000,
        "best_step": 128000,
        "train_seconds": 54122.43
      },
      "training_seed": 1,
      "batch_size": 128,
      "sequence_presentations": 16384000,
      "pipeline_seconds": 55543.618,
      "runtime": {
        "mlx": "0.32.3",
        "device": "Device(gpu, 0)",
        "metal": false,
        "cuda": true,
        "machine": "x86_64",
        "macos": "",
        "python": "3.12.3",
        "chip": "NVIDIA A40",
        "memory_gb": 0.0
      },
      "parameters": {
        "codec": 13915200,
        "generator": 6440128
      },
      "codec_validation": {
        "char_acc_by_levels_used": [
          0.1573,
          0.157,
          0.1578,
          0.1638,
          0.1635,
          0.2002,
          0.2593,
          0.3349,
          0.4487,
          0.5671,
          0.7239,
          1.0
        ],
        "seq_exact_by_levels_used": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          1.0
        ],
        "codebook_utilization": [
          1.0,
          0.9375,
          0.6875,
          0.625,
          0.5,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0
        ],
        "code_perplexity": [
          15.93,
          14.68,
          10.79,
          9.83,
          7.07,
          15.77,
          15.49,
          15.83,
          15.92,
          15.56,
          15.45,
          14.53
        ]
      },
      "generator_validation_teacher_forced": {
        "ce_per_level": [
          0.896,
          0.1557,
          0.41,
          0.2739,
          0.1699,
          0.336,
          0.1066,
          0.1613,
          0.1204,
          0.1451,
          0.1854,
          0.051
        ],
        "acc_per_level": [
          0.671,
          0.972,
          0.8427,
          0.891,
          0.9407,
          0.8723,
          0.9612,
          0.9383,
          0.9552,
          0.9429,
          0.9278,
          0.9802
        ]
      },
      "best_validation_weighted_ce": 0.1425576433121019,
      "generation_temperature": 0.0,
      "quality": {
        "val": {
          "solved": 0.71,
          "valid_grid": 0.712,
          "respects_givens": 0.997,
          "cell_acc": 0.9005,
          "blank_cell_acc": 0.8323,
          "n": 1000.0
        },
        "upstream-easy": {
          "solved": 0.9725,
          "valid_grid": 0.9725,
          "respects_givens": 1.0,
          "cell_acc": 0.9979,
          "blank_cell_acc": 0.9959,
          "n": 2000.0
        },
        "upstream-medium": {
          "solved": 0.796,
          "valid_grid": 0.796,
          "respects_givens": 0.999,
          "cell_acc": 0.9796,
          "blank_cell_acc": 0.9641,
          "n": 2000.0
        },
        "upstream-hard": {
          "solved": 0.3325,
          "valid_grid": 0.335,
          "respects_givens": 0.993,
          "cell_acc": 0.905,
          "blank_cell_acc": 0.8492,
          "n": 2000.0
        }
      },
      "benchmark": {
        "runtime": {
          "mlx": "0.32.3",
          "device": "Device(gpu, 0)",
          "metal": false,
          "cuda": true,
          "machine": "x86_64",
          "macos": "",
          "python": "3.12.3",
          "chip": "NVIDIA A40",
          "memory_gb": 0.0
        },
        "seq_len": 81,
        "levels": [
          1,
          2,
          4,
          6,
          9,
          13,
          18,
          26,
          37,
          50,
          67,
          81
        ],
        "temperature": 1.0,
        "warmup": 3,
        "iterations": 20,
        "sampler_compiled": false,
        "synchronization": "numpy_host_copy",
        "rows": [
          {
            "batch": 1,
            "first_call_ms": 518.31,
            "p50_ms": 29.453,
            "p95_ms": 30.323,
            "mean_ms": 29.243,
            "stdev_ms": 0.981,
            "sequences_per_s": 34.0,
            "chars_per_s": 2750.0,
            "peak_memory_mb": 130.2,
            "model_forwards": 12
          },
          {
            "batch": 16,
            "first_call_ms": 53.67,
            "p50_ms": 48.506,
            "p95_ms": 49.274,
            "mean_ms": 48.49,
            "stdev_ms": 0.708,
            "sequences_per_s": 329.9,
            "chars_per_s": 26718.0,
            "peak_memory_mb": 552.4,
            "model_forwards": 12
          },
          {
            "batch": 64,
            "first_call_ms": 157.03,
            "p50_ms": 174.326,
            "p95_ms": 175.352,
            "mean_ms": 173.246,
            "stdev_ms": 3.481,
            "sequences_per_s": 367.1,
            "chars_per_s": 29737.0,
            "peak_memory_mb": 1911.2,
            "model_forwards": 12
          },
          {
            "batch": 256,
            "first_call_ms": 596.91,
            "p50_ms": 581.791,
            "p95_ms": 583.397,
            "mean_ms": 581.875,
            "stdev_ms": 0.798,
            "sequences_per_s": 440.0,
            "chars_per_s": 35642.0,
            "peak_memory_mb": 2981.0,
            "model_forwards": 12
          }
        ]
      },
      "wilson95_solved": {
        "val": [
          0.681115,
          0.737278
        ],
        "upstream-easy": [
          0.964377,
          0.978812
        ],
        "upstream-medium": [
          0.77778,
          0.813085
        ],
        "upstream-hard": [
          0.312192,
          0.353451
        ]
      },
      "evidence_status": "Recorded aggregate evaluation; retained bytes verified, not independently rescored.",
      "limitations": [
        "No per-board predictions or checkpoint-bound ordered evaluation-input manifest were retained; local board hashes match the published files but remote evaluation identity and duplicate-free coverage are not independently verified.",
        "The CUDA benchmark GPU-load log is empty; preflight observations do not establish contention at benchmark time.",
        "Different physical A40 allocations; neither strict hardware identity nor cross-host timing attribution is established.",
        "Wilson intervals describe board sampling under independence assumptions, not training-seed uncertainty or correction for repeated benchmark-guided recipe selection."
      ]
    }
  }
}