{ "schema_version": 1, "status": "preliminary: seed-0 complete; seed-1 comparisons pending", "observed_at_utc": "2026-10-30T14:33:59.325235+00:00", "method": "Read-only inspection and CPU aggregation of retained evidence; no new model inference, training, timing, or LLM judge.", "source": { "training_commit": "d323f4f7803d4e3607dd21dd9d28ae41c1cce2ae", "training_and_optimized_sha256": { "training": { "ncm_mlx/__init__.py": "e5098865bd2e7b34d3e931f4580c51ff0c31a9d4fb02b8169ec614ed45f9b66c", "ncm_mlx/__main__.py": "bbc7d931025cffd6d3cd3a3c38d39356c226fa54cbd4aadbc42b0283f7bb2341", "ncm_mlx/baselines/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/baselines/ar.py": "4ef7d45a34b2ea61cab7cf5fc916fb0c3509af29d2cf8e01c3459ac5879588d0", "ncm_mlx/baselines/openai_llm.py": "7bf41c8a31f12942f87ed08b46fff344509fb8790d6bdffe03faac41f5ba3bc4", "ncm_mlx/cli.py": "6b85b5a3fd7295ea3254f4e8da79e86868a6d7048bf97bbc5ddb3a3de1ccb9ca", "ncm_mlx/config.py": "2656cf564437a95b963eabf7a12d2f977eaec4aef7261661b6bf4a4792f546ee", "ncm_mlx/device.py": "6d55e73760e5cf32342b7edf66fc555104c52310b79e85722f6658bcd347daf7", "ncm_mlx/eval/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/eval/benchmark.py": "63db8d781849eafa2845f6e7443f97bbbc4eb62e0851e4886aa68560207684a7", "ncm_mlx/eval/metrics.py": "a1ba0f8343ebd3ca3a52b64b0a01022e1c3cc8efaf5ec96f2f84e6efe7e3d2f3", "ncm_mlx/generation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/generation/sampler.py": "644751d3b96c298f41d8f606d66314477142fb65e640093135efa564dbf2270b", "ncm_mlx/latent/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/latent/codec.py": "7c5e95934874b79b7f15c8ed6baceb36c1c2c997be05479bde4a4c2a0bbcf895", "ncm_mlx/latent/quantizer.py": "a5fcf78884b5c8e77f9444fba37c1fd81f1e8204f4259868225b90efaeb3bbdf", "ncm_mlx/latent/resample.py": "53c8c1f124f20de6f57f7b817096be77ebae5f86eb45ad3730feb91362790fcd", "ncm_mlx/models/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/models/blocks.py": "7abeee08472e04e87173cd3d8b98b2e8b2d80fb2cc5bdde9e3ca5a5f667cc1ac", "ncm_mlx/models/generator.py": "65588be775a29aeb8878c3094ab6eb83059b1b1dd155f69c43fdebd5ef38a775", "ncm_mlx/official/__init__.py": "3eb778b0a73ddf6cb97f44536a8a21f8d16eb160317e193035965d66696e5033", "ncm_mlx/official/ar.py": "fc561626990d917736dd49a4108242a6c5339fefd70506ae5e9de0938ecb5ec4", "ncm_mlx/official/bpe.py": "4e23460a6fecc4ec1906f69679309d3198345f64682252c12397db2c85757414", "ncm_mlx/official/cli.py": "47732073b36e16100b1afe46bac71c10d0b5508f626f44543dc96e78988eab2a", "ncm_mlx/official/convert.py": "e06115a12f9ccf1f9962713066fc596751bded153b9600753bd762bc3794f2b3", "ncm_mlx/official/model.py": "a8b77cb8fc82f357d2419dcd0a5aa2a21f7ead2eadcc0bfcba20fc65aee7bb00", "ncm_mlx/official/sample.py": "36ab74963352310d4692c88b8e55f0135aeeea4b62c28627c18022b6d377edb7", "ncm_mlx/official/spec.py": "06cb71f6513e3065a346516b867794acbb87d42a06f1c9132974755a6d0ed4a1", "ncm_mlx/official/text_eval.py": "bf373883fe47631fee385771af542d0c74e37d665b1d459e651c4bbf8c9b0558", "ncm_mlx/official/text_run.py": "3b912f1194f38c856c56c9e36d3214e914601dede5bbf6a6f5914b617b88ad7e", "ncm_mlx/official/tinystories.py": "ffe2d7ae9dfec6161b09a002474bd73b5c0557ae2ed36e1a9dcc1dc31b17b8f3", "ncm_mlx/official/torch_reader.py": "7833d54cb33ea634958edd954bf6ba1a7986148a2773703288e85373c8f2f3b4", "ncm_mlx/official/train_config.py": "3ae1139a70b42dc3de2caac1251c94ab9d19bef71340150809b4f054afb0c71c", "ncm_mlx/official/trainer.py": "c097948b478e9e54cb6bc99fda0441901829d14f2fad5326885acc43a924c10e", "ncm_mlx/text/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/text/sudoku.py": "ce55c05cf5309a2671ff6ddbdac6cf7afdf0f28aef6f54e4c38b52dd772ce4f5", "ncm_mlx/text/tokenizer.py": "01754027d4f337c6ae273a6131e6abb105b93b23ccca4bab5c53cfe1d83ebdcd", "ncm_mlx/text/toygrammar.py": "e77b0277c5d0617454d2d343b04216e8a1bb5cc1526e6bef470c3d88db498b7e", "ncm_mlx/text/upstream_sudoku.py": "d9ea448eb6908475516170fce7b6712b25d0b646011de1f4a0a6ceb7428c467d", "ncm_mlx/training/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/training/checkpoint.py": "aa28244ab97839dc9100bf2d1dc9ca1ee4dde18bea52bf2ec218ddab91ce7d84", "ncm_mlx/training/codec.py": "4bb6f755559d403dec2d358c0d8c896856726921e1321ba521b44c785909cb87", "ncm_mlx/training/common.py": "aac71af21f6d2108554aae3ce92e328ef831911d4114e17fdaaf70b78d787d61", "ncm_mlx/training/generator.py": "70ba1276cbd09bae0420f68b162f075507e628d4686a45e2c340a5c61a9ee2dd" }, "optimized": { "ncm_mlx/__init__.py": "e5098865bd2e7b34d3e931f4580c51ff0c31a9d4fb02b8169ec614ed45f9b66c", "ncm_mlx/__main__.py": "bbc7d931025cffd6d3cd3a3c38d39356c226fa54cbd4aadbc42b0283f7bb2341", "ncm_mlx/baselines/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/baselines/ar.py": "4ef7d45a34b2ea61cab7cf5fc916fb0c3509af29d2cf8e01c3459ac5879588d0", "ncm_mlx/baselines/openai_llm.py": "7bf41c8a31f12942f87ed08b46fff344509fb8790d6bdffe03faac41f5ba3bc4", "ncm_mlx/cli.py": "6b85b5a3fd7295ea3254f4e8da79e86868a6d7048bf97bbc5ddb3a3de1ccb9ca", "ncm_mlx/config.py": "2656cf564437a95b963eabf7a12d2f977eaec4aef7261661b6bf4a4792f546ee", "ncm_mlx/device.py": "6d55e73760e5cf32342b7edf66fc555104c52310b79e85722f6658bcd347daf7", "ncm_mlx/eval/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/eval/benchmark.py": "63db8d781849eafa2845f6e7443f97bbbc4eb62e0851e4886aa68560207684a7", "ncm_mlx/eval/metrics.py": "a1ba0f8343ebd3ca3a52b64b0a01022e1c3cc8efaf5ec96f2f84e6efe7e3d2f3", "ncm_mlx/generation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/generation/sampler.py": "644751d3b96c298f41d8f606d66314477142fb65e640093135efa564dbf2270b", "ncm_mlx/latent/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/latent/codec.py": "7c5e95934874b79b7f15c8ed6baceb36c1c2c997be05479bde4a4c2a0bbcf895", "ncm_mlx/latent/quantizer.py": "a5fcf78884b5c8e77f9444fba37c1fd81f1e8204f4259868225b90efaeb3bbdf", "ncm_mlx/latent/resample.py": "53c8c1f124f20de6f57f7b817096be77ebae5f86eb45ad3730feb91362790fcd", "ncm_mlx/models/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/models/blocks.py": "7abeee08472e04e87173cd3d8b98b2e8b2d80fb2cc5bdde9e3ca5a5f667cc1ac", "ncm_mlx/models/generator.py": "65588be775a29aeb8878c3094ab6eb83059b1b1dd155f69c43fdebd5ef38a775", "ncm_mlx/official/__init__.py": "3eb778b0a73ddf6cb97f44536a8a21f8d16eb160317e193035965d66696e5033", "ncm_mlx/official/bpe.py": "4e23460a6fecc4ec1906f69679309d3198345f64682252c12397db2c85757414", "ncm_mlx/official/cli.py": "1b957a734eb50894fc625a5d2ad881fc8ce5e639fe3be28c5e63ce5600bfd6a9", "ncm_mlx/official/convert.py": "e06115a12f9ccf1f9962713066fc596751bded153b9600753bd762bc3794f2b3", "ncm_mlx/official/model.py": "41ab2fe4c4b45e46c62a2313b47aee511c5a22f7fb5183901d18dcff6e3e4229", "ncm_mlx/official/sample.py": "4c77f3d095807b7b5cd64803f9fdd75ad069b4bf5157fface07016062ddbcbf8", "ncm_mlx/official/spec.py": "06cb71f6513e3065a346516b867794acbb87d42a06f1c9132974755a6d0ed4a1", "ncm_mlx/official/torch_reader.py": "7833d54cb33ea634958edd954bf6ba1a7986148a2773703288e85373c8f2f3b4", "ncm_mlx/text/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/text/sudoku.py": "ce55c05cf5309a2671ff6ddbdac6cf7afdf0f28aef6f54e4c38b52dd772ce4f5", "ncm_mlx/text/tokenizer.py": "01754027d4f337c6ae273a6131e6abb105b93b23ccca4bab5c53cfe1d83ebdcd", "ncm_mlx/text/toygrammar.py": "e77b0277c5d0617454d2d343b04216e8a1bb5cc1526e6bef470c3d88db498b7e", "ncm_mlx/text/upstream_sudoku.py": "d9ea448eb6908475516170fce7b6712b25d0b646011de1f4a0a6ceb7428c467d", "ncm_mlx/training/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "ncm_mlx/training/checkpoint.py": "aa28244ab97839dc9100bf2d1dc9ca1ee4dde18bea52bf2ec218ddab91ce7d84", "ncm_mlx/training/codec.py": "4bb6f755559d403dec2d358c0d8c896856726921e1321ba521b44c785909cb87", "ncm_mlx/training/common.py": "aac71af21f6d2108554aae3ce92e328ef831911d4114e17fdaaf70b78d787d61", "ncm_mlx/training/generator.py": "70ba1276cbd09bae0420f68b162f075507e628d4686a45e2c340a5c61a9ee2dd" } }, "optimized_revision": "Frozen content hashes identify the benchmark implementation; a public source snapshot is required before release." }, "evaluation_contract": { "samples_per_seed": 256, "seeds": [ 42, 0, 1, 2, 3 ], "batch": 128, "test_rows": 4096, "temperature": 1, "top_k": 0, "top_p": 1, "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519" }, "benchmark_contract": { "batch_sizes": [ 1, 16, 64 ], "warmup": 3, "iterations": 20, "temperature": 1, "synchronization": "Decoded tokens copied to NumPy host array" }, "experiments": { "e21": { "name": "tinystories-e21", "normalized_config": { "name": "tinystories-e21", "out_dir": "artifacts/tinystories-e21", "data": { "seq_len": 128, "raw_dir": "artifacts/tinystories/raw", "cache_root": "artifacts/tinystories", "bpe": "artifacts/official/tinystories/mlx/bpe.json", "limit": 0, "val_rows": 2048, "unigram_rows": 200000 }, "codec": { "dim": 256, "n_head": 4, "levels": [ 1, 8, 16, 24, 32, 40, 52, 64, 76, 92, 108, 128 ], "codebook_sizes": [ 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048 ], "l0_strides": [ 4, 4, 4, 2 ], "group_norm_groups": 8, "phi_ratio": 0.5, "dropout": 0.3, "decay": 0.95, "epsilon": 1e-05, "commitment": 0.25, "fine_drop_prob": 0.5, "fine_drop_min_keep": 1, "restart_threshold": 0.01 }, "generator": { "n_layer": 6, "n_head": 6, "dim": 384, "rope_base": 100000.0, "dropout": 0.1, "corrupt_first": 0.75, "corrupt_last": 0.25, "loss_alpha": 0.0 }, "train": { "seed": 0, "log_every": 200, "eval_every": 1000, "compile": true, "codec": { "steps": 14000, "batch_size": 256, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 1000, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.01, "grad_clip": 1.0, "max_minutes": 120 }, "generator": { "steps": 15000, "batch_size": 64, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 390 }, "ar": { "steps": 7500, "batch_size": 128, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 90 } } }, "config_file_sha256": "5ee072df377ca44fc0d046971085908f1175357122190731d5a97e2c4f94462d", "report": { "logical_path": "runs/tinystories-e21-aligned-eval/report.json", "sha256": "52213efdc9fc2a550b875c0e99379a341b7caf1510dbd72cd15d943e5bcbbb14", "bytes": 31208, "started_local": "2026-10-30 05:29:36", "finished_local": "2026-10-30 05:33:24", "wall_seconds": 228.40949702262878 }, "runtime": { "mlx": "0.32.3", "device": "Device(gpu, 0)", "metal": true, "cuda": false, "machine": "arm64", "macos": "26.5", "python": "3.12.14", "chip": "Apple M5 Pro", "memory_gb": 64.0 }, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "identities": { "ordered_test_windows_sha256": "ae33fdb4221503283c5853f7639c163060a65d22a68f174883fb7d84baf2b66c", "data_manifest_sha256": "c576dbb4a51a290abb17532a516564a8bfd4ec5e5d61e2bb3a6da1888063490f", "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "4aef782609f484696271d2896acdae8f25389cfc120a7c21182d79b8f34441d6", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "73fc4adfb4dae4b59d342bfb89b49e08abaff0aa7e8c0f7db1aaf735b2249f17", "bytes": 57413 } }, "ar_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } }, "official_files": { "codec.safetensors": { "sha256": "b28e6c7dc804fcd056ca98ff8dc818484a8bb17d52169c88205eb3d27b74c593", "bytes": 145745190 }, "generator.safetensors": { "sha256": "e24e9ddde6b616add427420d7da97d86e92fbf1199a187dfc2e005fdeff5c49d", "bytes": 354440547 }, "inference.safetensors": { "sha256": "5fa49c2b71abe94319277d5aa020719ab9e781880e6d2e8347315c8914d74163", "bytes": 16486 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "e9cf868d06141400d061a8bee2e8d95df541fb90548d0b3e0d43e314f1479b1f", "bytes": 3067 } }, "scorer_files": { "model.safetensors": { "sha256": "5f47f3e12f91cd33b662ce7e433b6150ad5512b5884a2cee961b50e9c3bbebce", "bytes": 3247159078 }, "config.json": { "sha256": "7fccdcfd6622055342a734c663ee0b61ff4fd697f42467595df0bf4448c8c170", "bytes": 666 }, "tokenizer.json": { "sha256": "8414cab924d8b9b33013f0d221c5862f365ee9be39c5c2bfae8a5a9e970478a6", "bytes": 1355256 } }, "job_driver_sha256": "3afe1757667ca196cf6b7d33fc27470d6c74323c19850c9590e4c482e49fe71f", "plan_sha256": "ecc669dbc593113b3143e9114abc5a37973a53da26080724fa47bd4e2d838fc2" }, "training": { "codec": { "steps": 14000, "planned_steps": 14000, "time_limited": false, "train_seconds": 5517.25, "sequences_seen": 3584000, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "val": { "sequences": 2048, "token_acc": 0.99819, "nonpad_token_acc": 0.99819, "exact_window_pct": 84.814, "recon_ce": 0.00954, "cumulative_nonpad_acc": [ 0.1325, 0.1716, 0.2614, 0.3757, 0.5327, 0.6336, 0.7484, 0.9008, 0.9377, 0.9609, 0.9756, 0.9982 ], "code_usage": [ 0.5918, 0.6001, 0.9204, 0.9805, 0.9873, 0.957, 0.9155, 0.981, 0.8838, 0.8584, 0.8848, 0.9961 ] }, "level0_codes_used": 1987 }, "generator": { "steps": 15000, "planned_steps": 15000, "time_limited": false, "train_seconds": 13898.17, "sequences_seen": 960000, "corruption_rates": [ 0.75, 0.7, 0.65, 0.6, 0.55, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25 ], "val": { "code_ce_per_level": [ 6.6309, 5.1109, 4.3014, 3.739, 3.3933, 2.7008, 1.3188, 2.1075, 2.4508, 2.4911, 0.5365 ], "code_ce_position_weighted": 2.25255 } }, "ar_common_control": { "steps": 7500, "planned_steps": 7500, "time_limited": false, "train_seconds": 2062.03, "sequences_seen": 960000, "parameters": 13794048 } }, "models": { "data": { "note": "held-out test windows, decoded with the same cleanup; disjoint per seed", "diagnostics": { "samples": 1280, "mean_tokens": 127.32, "mean_chars": 511.9, "eos_in_window_pct": 5.94, "empty_pct": 0.0, "unigram_entropy": 3.9519, "duplicate_samples_pct": 0.0, "distinct_1": 0.025, "distinct_2": 0.2137, "distinct_3": 0.485, "repeated_4gram_pct": 1.01, "copied_train_opening_32_pct": 4.61 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 8.587, "0": 8.612, "1": 8.486, "2": 8.504, "3": 8.457 }, "mean": 8.529, "sd": 0.067, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } }, "official": { "codec_test": { "sequences": 4096, "token_acc": 0.99536, "nonpad_token_acc": 0.99981, "exact_window_pct": 92.969, "recon_ce": 0.06758, "cumulative_nonpad_acc": [ 0.0976, 0.1179, 0.2212, 0.363, 0.558, 0.6954, 0.824, 0.9416, 0.9678, 0.982, 0.9898, 0.9998 ], "code_usage": [ 0.3975, 0.75, 0.9443, 0.96, 0.9622, 0.9199, 0.9048, 0.9597, 0.9209, 0.9375, 0.9548, 0.9431 ] }, "generator_test_code_ce": { "code_ce_per_level": [ 6.9376, 5.017, 3.5519, 2.4696, 1.6623, 1.2494, 0.473, 1.1195, 1.4427, 1.5027, 0.2744 ], "code_ce_position_weighted": 1.37032 }, "parameters": { "codec": 36434816, "generator": 88605707 }, "model_forwards_per_sample": 11, "diagnostics": { "samples": 1280, "mean_tokens": 128.0, "mean_chars": 510.0, "eos_in_window_pct": 0.31, "empty_pct": 0.0, "unigram_entropy": 3.9699, "duplicate_samples_pct": 0.0, "distinct_1": 0.0195, "distinct_2": 0.194, "distinct_3": 0.4595, "repeated_4gram_pct": 0.58, "copied_train_opening_32_pct": 0.7 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 16.73, "0": 16.396, "1": 15.921, "2": 16.183, "3": 16.064 }, "mean": 16.259, "sd": 0.315, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } }, "ncm": { "codec_test": { "sequences": 4096, "token_acc": 0.99816, "nonpad_token_acc": 0.99816, "exact_window_pct": 84.937, "recon_ce": 0.00992, "cumulative_nonpad_acc": [ 0.1339, 0.1718, 0.2629, 0.377, 0.5347, 0.6351, 0.7495, 0.9017, 0.9376, 0.9614, 0.9764, 0.9982 ], "code_usage": [ 0.813, 0.6187, 0.9307, 0.9844, 0.9951, 0.9858, 0.9609, 0.9912, 0.9336, 0.9238, 0.9316, 0.999 ] }, "generator_test_code_ce": { "code_ce_per_level": [ 6.6354, 5.0934, 4.2807, 3.739, 3.3739, 2.7109, 1.3148, 2.1052, 2.4546, 2.4818, 0.5337 ], "code_ce_position_weighted": 2.24875 }, "parameters": { "codec": 15508096, "generator": 11832587 }, "model_forwards_per_sample": 11, "diagnostics": { "samples": 1280, "mean_tokens": 127.36, "mean_chars": 502.5, "eos_in_window_pct": 7.11, "empty_pct": 0.0, "unigram_entropy": 3.9349, "duplicate_samples_pct": 0.0, "distinct_1": 0.0191, "distinct_2": 0.2041, "distinct_3": 0.4951, "repeated_4gram_pct": 0.51, "copied_train_opening_32_pct": 0.0 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 48.559, "0": 46.974, "1": 48.46, "2": 48.412, "3": 47.08 }, "mean": 47.897, "sd": 0.797, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } }, "ar": { "ar_test": { "token_ce": 1.31454, "nonpad_token_ce": 1.32042 }, "parameters": { "ar": 13794048 }, "model_forwards_per_sample": 128, "diagnostics": { "samples": 1280, "mean_tokens": 127.61, "mean_chars": 512.9, "eos_in_window_pct": 3.12, "empty_pct": 0.0, "unigram_entropy": 3.9854, "duplicate_samples_pct": 0.0, "distinct_1": 0.0251, "distinct_2": 0.2184, "distinct_3": 0.491, "repeated_4gram_pct": 0.84, "copied_train_opening_32_pct": 1.25 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 14.885, "0": 14.581, "1": 14.032, "2": 15.258, "3": 14.623 }, "mean": 14.676, "sd": 0.45, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } } }, "scorer_records": { "sha256": "c0435ed77bef0d3ae48e3e4c222c1142575431c5b57ca61bacabf895575b04c5", "bytes": 1085442 }, "samples": { "data": { "sha256": "153109479bffd94014627800257cd8e0f84e6cc0adc6c2445cbddf32bf49574d", "bytes": 1564511 }, "official": { "sha256": "9ff6dccc2d47a39213e60a01216b7365cb8c14b3dd8b495500d0d731b278f40c", "bytes": 1565254 }, "ncm": { "sha256": "a002648ac64a8d5aabfc6a834f16eb4261cb11a68f9eabb92b9df43415e18c6d", "bytes": 1546306 }, "ar": { "sha256": "718aeec386f0da4ab75f580c1bc9ba1d5eee1ce6b934f545c709047a7ebc4890", "bytes": 1565298 } }, "first_eight_ncm_samples_seed_42": [ { "model": "ncm", "seed": 42, "index": 0, "text": "\n\nOnce there was a little boy called Jack. Joe was was three years old and discover. One One day, he wanted to go and look for exploring. \nHe started to explore.\nAs was walking in the park, he saw a big giant giant on a frog. Jack looked at the neare and said, \"Why, do you think? Are do do?\" \nJohn said, \"Yes, but we have to small, small.\" He started to walk the He and waited for something fun to do. \nWhen they got there, Joe he listened listened until he slowly. As as he onto the hole", "ids": [ 12, 12, 438, 408, 284, 260, 398, 505, 1027, 1218, 48, 1723, 284, 284, 1487, 1902, 788, 268, 2835, 48, 536, 536, 330, 46, 287, 415, 267, 489, 268, 709, 395, 2829, 48, 34, 12, 1033, 557, 267, 1432, 48, 12, 925, 284, 1286, 316, 264, 530, 46, 287, 390, 260, 348, 2216, 2216, 355, 260, 981, 48, 1218, 512, 457, 264, 7607, 268, 327, 46, 320, 1315, 46, 382, 351, 1133, 65, 4220, 382, 382, 485, 34, 12, 1716, 327, 46, 320, 744, 46, 416, 366, 508, 267, 568, 46, 568, 403, 319, 557, 267, 962, 264, 319, 268, 1544, 395, 586, 451, 267, 382, 48, 34, 12, 1043, 364, 665, 408, 46, 1723, 287, 1369, 1369, 983, 287, 2074, 48, 1343, 456, 287, 2461, 264, 1090 ] }, { "model": "ncm", "seed": 42, "index": 1, "text": "\n\n a a time, there was a large black dog named Max. Max lived in a small house with many soft toys. One day, Max liked to serve and play with the mud by the mud.\nOne day, Max was playing in a big car named a saw all all the mud. He wanted to look to look at the things he saw him. her told her \"Hello, Ben for a gift for for our friend!\" Tom said, \"No, I can't find you you do do.\"\nTim wanted to the gift with his family. He said, \"I don't you. Please laser back!\" Ben", "ids": [ 12, 12, 260, 260, 404, 46, 408, 284, 260, 2474, 1618, 474, 409, 523, 48, 523, 640, 316, 260, 568, 612, 331, 670, 1039, 621, 48, 536, 330, 46, 523, 537, 267, 4276, 268, 328, 331, 264, 1770, 862, 264, 1770, 48, 12, 391, 330, 46, 523, 284, 592, 316, 260, 348, 533, 409, 260, 390, 440, 440, 264, 1770, 48, 319, 415, 267, 709, 267, 709, 457, 264, 633, 287, 390, 483, 48, 338, 747, 338, 320, 1188, 46, 560, 395, 260, 1802, 395, 395, 1150, 359, 421, 423, 327, 46, 320, 943, 46, 341, 476, 501, 579, 351, 351, 382, 382, 403, 12, 655, 415, 267, 264, 1802, 331, 345, 953, 48, 319, 327, 46, 320, 75, 869, 501, 351, 48, 2589, 3969, 521, 421, 560 ] }, { "model": "ncm", "seed": 42, "index": 2, "text": ",\n\nTim and Lily were twins friends who liked to play with their toys. One day, Max saw a big dog dog big red lock on a lock. Tim was very happy. He thought the dog was key.\nTom wanted to escape each other. He thought of of a plan to get a big tree. He thought the stick was a stick and then then threw it away. But Tom did not know. He had a bag in the bag.\nTom went to see Tom. When he touched his ball, he knocked the door. door pushed, and they were both surprised. They did not know this key was on", "ids": [ 46, 12, 12, 655, 268, 413, 412, 1978, 419, 667, 537, 267, 328, 331, 477, 621, 48, 536, 330, 46, 523, 390, 260, 348, 474, 474, 348, 678, 2466, 355, 260, 2466, 48, 344, 284, 386, 384, 48, 319, 589, 264, 474, 284, 1607, 48, 12, 743, 415, 267, 3093, 765, 562, 48, 319, 589, 373, 373, 260, 1374, 267, 653, 260, 348, 506, 48, 319, 589, 264, 988, 284, 260, 988, 268, 642, 642, 1594, 312, 581, 48, 548, 423, 460, 368, 663, 48, 319, 367, 260, 1246, 316, 264, 1246, 48, 12, 743, 435, 267, 547, 423, 48, 942, 287, 2011, 345, 484, 46, 287, 2726, 264, 911, 48, 911, 1420, 46, 268, 364, 412, 728, 902, 48, 315, 460, 368, 663, 737, 1607, 284, 355 ] }, { "model": "ncm", "seed": 42, "index": 3, "text": " them and lady was,, in a small little girl there Lily a Lily was hot. She loved to play outside. One day, Lily did not play outside. She put on her hat and felt a warm scarf.\nOne sunny day, Lily, came back to the garden. She saw Lily was and said. \"\"No, Lily!\" want said, \"Lily, why are tired?\" Lily said,, \"I need to stay warm. But I want to be warm and and feel..\"\nLily started to feel better and smiled. She gave her a big hug. Lily gave Lily better and felt feel better. She", "ids": [ 499, 268, 1305, 284, 46, 46, 316, 260, 568, 398, 486, 408, 413, 260, 413, 284, 1278, 48, 342, 511, 267, 328, 707, 48, 536, 330, 46, 413, 460, 368, 328, 707, 48, 342, 583, 355, 338, 1026, 268, 543, 260, 1050, 2524, 48, 12, 391, 1167, 330, 46, 413, 46, 559, 521, 267, 264, 912, 48, 342, 390, 413, 284, 268, 327, 48, 320, 36, 943, 46, 413, 421, 372, 327, 46, 320, 674, 46, 1482, 492, 1140, 485, 413, 327, 46, 46, 320, 75, 778, 267, 971, 1050, 48, 548, 341, 372, 267, 325, 1050, 268, 268, 742, 48, 403, 12, 674, 557, 267, 742, 831, 268, 571, 48, 342, 766, 338, 260, 348, 692, 48, 413, 766, 413, 831, 268, 543, 742, 831, 48, 342 ] }, { "model": "ncm", "seed": 42, "index": 4, "text": "able with a a time, in a big gray town, there lived the cat named Tom. Tom liked to climb the tree, but he was too small to what would happen. He would climb very high shelf on top,, down and down.\n\nAt the top of the top, a little bird Tom came to him and asked, \"Let's go touch was ladder!\" Tom looked at her and said, \"I can see enough the can climb very high and high on the ladder\nAs Tom played, but he was not easy. So, he swung up and fast, and he couldn not see it. Tom reached out", "ids": [ 841, 331, 260, 260, 404, 46, 316, 260, 348, 2109, 1013, 46, 408, 640, 264, 468, 409, 423, 48, 423, 537, 267, 918, 264, 506, 46, 416, 287, 284, 430, 568, 267, 585, 538, 1692, 48, 319, 538, 918, 386, 905, 2243, 355, 1292, 46, 46, 613, 268, 613, 48, 12, 12, 1224, 264, 1292, 373, 264, 1292, 46, 260, 398, 426, 423, 559, 267, 483, 268, 550, 46, 320, 904, 392, 489, 1342, 284, 2490, 421, 423, 512, 457, 338, 268, 327, 46, 320, 75, 476, 547, 1773, 264, 476, 918, 386, 905, 268, 905, 355, 264, 2490, 12, 925, 423, 482, 46, 416, 287, 284, 368, 2383, 48, 722, 46, 287, 4355, 370, 268, 739, 46, 268, 287, 1063, 368, 547, 312, 48, 423, 1400, 493 ] }, { "model": "ncm", "seed": 42, "index": 5, "text": "\nOne day a, little girl named Lily went for a walk walk. She saw a big tree in a tree. The cat wanted to climb. She saw little bird in the tree. She said, \"Don't help you the cat, Lily!\"\nLily walked up to the tree and said, \"We need to reach first first. But, first!\" Lily felt sad. She and looked, but the cat could not reach it. Lily had an idea. She asked her friend, \"That is for the top called Lily's story,\" the bird said. \"It's save we will have fun fun,\" said the bird said", "ids": [ 12, 391, 330, 260, 46, 398, 486, 409, 413, 435, 395, 260, 962, 962, 48, 342, 390, 260, 348, 506, 316, 260, 506, 48, 286, 468, 415, 267, 918, 48, 342, 390, 398, 426, 316, 264, 506, 48, 342, 327, 46, 320, 1248, 501, 441, 351, 264, 468, 46, 413, 421, 12, 674, 796, 370, 267, 264, 506, 268, 327, 46, 320, 1152, 778, 267, 1477, 1182, 1182, 48, 548, 46, 1182, 421, 413, 543, 503, 48, 342, 268, 512, 46, 416, 264, 468, 475, 368, 1477, 312, 48, 413, 367, 428, 830, 48, 342, 550, 338, 359, 46, 320, 1105, 439, 395, 264, 1292, 1027, 413, 392, 1009, 646, 264, 426, 327, 48, 320, 1022, 392, 2341, 366, 625, 508, 451, 451, 646, 327, 264, 426, 327 ] }, { "model": "ncm", "seed": 42, "index": 6, "text": "\nOne day, a little girl named Sally found a thermometer in her room. She was very happy and opened the jar and put it on it. She mirror it it smell taste good. She ran to the kitchen kitchen and showed her mom a secret door on the door.\n\"Can we help open the door gate?\"\nHer mom smiled and.. \"But it is broken. She think it to know more more. persistent, it has to home back.\"\nSally and didn't give, she gave the prunes from her got. Her mom gave Lucy and told the lotion that the lotion was very kind and gave her lotion to", "ids": [ 12, 391, 330, 46, 260, 398, 486, 409, 1049, 514, 260, 4401, 316, 338, 716, 48, 342, 284, 386, 384, 268, 915, 264, 1762, 268, 583, 312, 355, 312, 48, 342, 2609, 312, 312, 1388, 2225, 567, 48, 342, 602, 267, 264, 1173, 1173, 268, 1005, 338, 394, 260, 1712, 911, 355, 264, 911, 48, 12, 36, 1035, 366, 441, 1040, 264, 911, 2630, 485, 12, 1585, 394, 571, 268, 48, 48, 320, 780, 312, 439, 1356, 48, 342, 1133, 312, 267, 663, 684, 684, 48, 3388, 46, 312, 1160, 267, 591, 521, 403, 12, 2501, 268, 813, 501, 926, 46, 399, 766, 264, 4874, 587, 338, 665, 48, 940, 394, 766, 693, 268, 747, 264, 3994, 389, 264, 3994, 284, 386, 805, 268, 766, 338, 3994, 267 ] }, { "model": "ncm", "seed": 42, "index": 7, "text": " the\n. a time, in a big green field, there lived a cheerful bird. The bird loved to sing and sing songs. It sang a beautiful songs all day.\nOne sunny day, a little bird named Bella came to the field. She wanted to sing a pretty song. Bella looked at the bird and said, \"Hello, little bird!! I music music! Can I like like song too?\" Bella thought about the bird bird happy and said, \"Yes, Bobo!\"\nBella and Bella played the songs and promised each each other. They became good friends. They had fun songs and played together. And they all", "ids": [ 264, 12, 48, 260, 404, 46, 316, 260, 348, 1069, 1729, 46, 408, 640, 260, 3163, 426, 48, 286, 426, 511, 267, 1122, 268, 1122, 2205, 48, 432, 1740, 260, 1000, 2205, 440, 330, 48, 12, 391, 1167, 330, 46, 260, 398, 426, 409, 2102, 559, 267, 264, 1729, 48, 342, 415, 267, 1122, 260, 758, 1296, 48, 2102, 512, 457, 264, 426, 268, 327, 46, 320, 1188, 46, 398, 426, 35, 35, 341, 1626, 1626, 35, 1129, 341, 525, 525, 1296, 430, 485, 2102, 589, 673, 264, 426, 426, 384, 268, 327, 46, 320, 744, 46, 1370, 421, 12, 5123, 268, 2102, 482, 264, 2205, 268, 1748, 765, 765, 562, 48, 315, 698, 567, 419, 48, 315, 367, 451, 2205, 268, 482, 466, 48, 726, 364, 440 ] } ], "recomputed_ppl": { "data": { "per_seed": { "42": 8.587192549995347, "0": 8.612104270271919, "1": 8.486438723526838, "2": 8.50383387571238, "3": 8.457419850820951 }, "mean": 8.529397854065486, "sd_sampling_seed": 0.0668214318094909, "max_scored_tokens": 145, "unscorable": 0 }, "official": { "per_seed": { "42": 16.729715291347294, "0": 16.3963302815983, "1": 15.921392481000778, "2": 16.183235359841557, "3": 16.064051800621353 }, "mean": 16.258945042881855, "sd_sampling_seed": 0.3154981753809111, "max_scored_tokens": 151, "unscorable": 0 }, "ncm": { "per_seed": { "42": 48.559037419727865, "0": 46.974455607290125, "1": 48.460058087317584, "2": 48.41211088042779, "3": 47.079540180871625 }, "mean": 47.897040435126996, "sd_sampling_seed": 0.7968682259607903, "max_scored_tokens": 155, "unscorable": 0 }, "ar": { "per_seed": { "42": 14.885483249080956, "0": 14.580821038376598, "1": 14.031941107377678, "2": 15.25808914998773, "3": 14.623375352318366 }, "mean": 14.675941979428266, "sd_sampling_seed": 0.4498197524543124, "max_scored_tokens": 152, "unscorable": 0 } }, "benchmarks": { "ncm": { "file": "tinystories-e21-aligned-bench-ncm.json", "sha256": "e35fefd9dbe00d2104c2d12c1d4c837214b0a45260fa6939b3db40518b911ef3", "bytes": 14266, "modes": { "eager": [ { "batch": 1, "first_call_ms": 28.730500023812056, "p50_ms": 12.790813110768795, "p95_ms": 13.005168735980988, "mean_ms": 12.812364636920393, "stdev_ms": 0.10421238323802294, "sequences_per_s": 78.18111259542081, "peak_memory_mb": 184.58904933929443, "raw_ms": [ 13.05112475529313, 12.999000027775764, 12.802292127162218, 12.84920796751976, 13.002749998122454, 12.736374977976084, 12.847792357206345, 12.916041072458029, 12.730208225548267, 12.716041877865791, 12.715667020529509, 12.732625007629395, 12.798875104635954, 12.7768749371171, 12.69437512382865, 12.731332797557116, 12.79341708868742, 12.746250256896019, 12.78820913285017, 12.818832881748676 ], "gpu_before": [ 0 ], "gpu_after": [ 84 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/eager-batch1.npy", "first_output_sha256": "14353e943d3f7e4273427c5e79ce93a5ef4fdff9bd739ce25b61c17b18df3a06" }, { "batch": 16, "first_call_ms": 54.47620898485184, "p50_ms": 45.431937323883176, "p95_ms": 45.888154744170606, "mean_ms": 45.550441718660295, "stdev_ms": 0.24054050359222207, "sequences_per_s": 352.1751644869641, "peak_memory_mb": 1114.458779335022, "raw_ms": [ 45.39795778691769, 45.88104225695133, 46.02329200133681, 45.786917209625244, 45.65295809879899, 45.747249852865934, 45.839833095669746, 45.87850021198392, 45.70483369752765, 45.56679166853428, 45.43212475255132, 45.33133329823613, 45.392374973744154, 45.36116681993008, 45.342667028307915, 45.26662500575185, 45.38791626691818, 45.262791216373444, 45.431749895215034, 45.320709235966206 ], "gpu_before": [ 0 ], "gpu_after": [ 95 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/eager-batch16.npy", "first_output_sha256": "84987bdd636621209784413f71faa33c2ec2bbc61679e94d32286fa907d4d8d9" }, { "batch": 64, "first_call_ms": 243.53674985468388, "p50_ms": 223.878167103976, "p95_ms": 227.09023356437683, "mean_ms": 224.52305231709033, "stdev_ms": 3.1985722941420835, "sequences_per_s": 285.8697693834361, "peak_memory_mb": 2256.573137283325, "raw_ms": [ 236.9145001284778, 223.53604203090072, 224.11666717380285, 222.12245920673013, 223.43687526881695, 224.24737503752112, 223.9835001528263, 222.58054185658693, 222.86879178136587, 221.18058428168297, 223.7728340551257, 225.1146249473095, 224.42662483081222, 225.60908319428563, 224.94683414697647, 223.00583263859153, 226.57316690310836, 225.85387527942657, 223.1765417382121, 222.9942916892469 ], "gpu_before": [ 0 ], "gpu_after": [ 93 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/eager-batch64.npy", "first_output_sha256": "82828256e8f67897c175db033d826c4e56b07a3e70a1aac3d7755257d35f3524" } ], "cached": [ { "batch": 1, "first_call_ms": 15.496042091399431, "p50_ms": 13.173207873478532, "p95_ms": 13.424774585291743, "mean_ms": 13.167923036962748, "stdev_ms": 0.19765677091393144, "sequences_per_s": 75.91165413955765, "peak_memory_mb": 182.86491870880127, "raw_ms": [ 13.370709028095007, 13.509958051145077, 13.148665893822908, 13.119792100042105, 13.254625257104635, 13.07870913296938, 13.197749853134155, 13.374958653002977, 13.370541855692863, 13.225000351667404, 13.108958955854177, 12.986874673515558, 13.03529180586338, 12.838250026106834, 12.78387475758791, 12.916042003780603, 13.253750279545784, 13.420291244983673, 13.310708105564117, 13.053708709776402 ], "gpu_before": [ 0 ], "gpu_after": [ 83 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached-batch1.npy", "first_output_sha256": "14353e943d3f7e4273427c5e79ce93a5ef4fdff9bd739ce25b61c17b18df3a06" }, { "batch": 16, "first_call_ms": 60.087790712714195, "p50_ms": 47.42395807988942, "p95_ms": 52.79358972329646, "mean_ms": 48.44844795297831, "stdev_ms": 2.2260964595336783, "sequences_per_s": 337.38221455591565, "peak_memory_mb": 1112.5509853363037, "raw_ms": [ 50.58758286759257, 52.790166810154915, 52.85862507298589, 52.07445891574025, 50.38687493652105, 49.76691724732518, 49.26791740581393, 47.71333280950785, 48.32033300772309, 47.15633438900113, 47.22533328458667, 47.18720819801092, 47.21758281812072, 47.110790852457285, 47.622582875192165, 46.707583125680685, 46.16154218092561, 45.92237528413534, 46.4372499845922, 46.4541669934988 ], "gpu_before": [ 0 ], "gpu_after": [ 96 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached-batch16.npy", "first_output_sha256": "b61731c038ad9e0a82a5d3d0e61c9e05aa8ed46a639113b44ff1ba24a29fac90" }, { "batch": 64, "first_call_ms": 261.11958315595984, "p50_ms": 231.75006243400276, "p95_ms": 237.38331701606512, "mean_ms": 232.3273291811347, "stdev_ms": 3.2814261118088357, "sequences_per_s": 276.15958040238183, "peak_memory_mb": 2413.095678329468, "raw_ms": [ 228.83466677740216, 237.15950036421418, 236.8343328125775, 233.94254175946116, 233.35324972867966, 233.51891618221998, 233.0796248279512, 231.66766576468945, 230.11554218828678, 227.8508748859167, 233.00474975258112, 229.93004182353616, 241.63583340123296, 232.58170811459422, 229.16958387941122, 231.83245910331607, 229.63391710072756, 231.17075022310019, 230.36687495186925, 230.8637499809265 ], "gpu_before": [ 0 ], "gpu_after": [ 93 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached-batch64.npy", "first_output_sha256": "79888bf5f2b87dd3d6af41f2d2a19a17fe345f7703a383bbc2a30efc1cd7892c" } ], "cached+compiled": [ { "batch": 1, "first_call_ms": 1647.8655408136547, "p50_ms": 11.651291511952877, "p95_ms": 11.889693094417453, "mean_ms": 11.63640629965812, "stdev_ms": 0.18247270337725977, "sequences_per_s": 85.82739509814132, "peak_memory_mb": 183.23144245147705, "raw_ms": [ 11.8241673335433, 12.044582981616259, 11.836250312626362, 11.766250245273113, 11.8815409950912, 11.760750319808722, 11.723708361387253, 11.651332955807447, 11.672249995172024, 11.704583652317524, 11.651250068098307, 11.613292153924704, 11.449292302131653, 11.407915968447924, 11.475541163235903, 11.472583282738924, 11.474375147372484, 11.442292015999556, 11.412082705646753, 11.464084032922983 ], "gpu_before": [ 0 ], "gpu_after": [ 77 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached+compiled-batch1.npy", "first_output_sha256": "14353e943d3f7e4273427c5e79ce93a5ef4fdff9bd739ce25b61c17b18df3a06" }, { "batch": 16, "first_call_ms": 56.958042085170746, "p50_ms": 42.86477086134255, "p95_ms": 44.03402879834175, "mean_ms": 42.7539853611961, "stdev_ms": 0.9639894683339221, "sequences_per_s": 373.2668967660235, "peak_memory_mb": 1150.341070175171, "raw_ms": [ 44.27129216492176, 43.50987495854497, 44.02154125273228, 43.713000137358904, 43.41920791193843, 43.45716629177332, 43.20562491193414, 43.44933293759823, 43.40212466195226, 42.66954120248556, 42.97850001603365, 42.75104170665145, 42.45000006631017, 42.57279168814421, 41.341624688357115, 41.31945874541998, 42.168083135038614, 42.02404199168086, 41.433666832745075, 40.921791922301054 ], "gpu_before": [ 0 ], "gpu_after": [ 95 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached+compiled-batch16.npy", "first_output_sha256": "b61731c038ad9e0a82a5d3d0e61c9e05aa8ed46a639113b44ff1ba24a29fac90" }, { "batch": 64, "first_call_ms": 226.2063748203218, "p50_ms": 201.30404154770076, "p95_ms": 205.56673747487366, "mean_ms": 202.25548527669162, "stdev_ms": 2.9082947514381003, "sequences_per_s": 317.9270495909772, "peak_memory_mb": 2551.962986946106, "raw_ms": [ 210.98862495273352, 205.20758302882314, 203.09179229661822, 200.4742082208395, 199.23808285966516, 200.86554111912847, 198.78000020980835, 201.06858341023326, 198.23212502524257, 200.41899988427758, 201.03016681969166, 200.98324958235025, 200.36587538197637, 201.53949968516827, 204.03958298265934, 205.28137497603893, 204.19366611167789, 204.29575024172664, 202.20966590568423, 202.80533283948898 ], "gpu_before": [ 0 ], "gpu_after": [ 95 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ncm/cached+compiled-batch64.npy", "first_output_sha256": "79888bf5f2b87dd3d6af41f2d2a19a17fe345f7703a383bbc2a30efc1cd7892c" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison. Some optimized GPU outputs differ from eager; this timing comparison is not a quality-equivalence claim.", "all_cold_outputs_equal_eager": false, "cold_output_consistency": [ { "mode": "eager", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 16, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 64, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 16, "tokens_equal": false, "different_token_fraction": 0.00048828125 }, { "mode": "cached", "batch": 64, "tokens_equal": false, "different_token_fraction": 0.003173828125 }, { "mode": "cached+compiled", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached+compiled", "batch": 16, "tokens_equal": false, "different_token_fraction": 0.00048828125 }, { "mode": "cached+compiled", "batch": 64, "tokens_equal": false, "different_token_fraction": 0.003173828125 } ], "source": "frozen optimized MLX sampler", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "4aef782609f484696271d2896acdae8f25389cfc120a7c21182d79b8f34441d6", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "73fc4adfb4dae4b59d342bfb89b49e08abaff0aa7e8c0f7db1aaf735b2249f17", "bytes": 57413 } } }, "ar": { "file": "tinystories-e21-aligned-bench-ar.json", "sha256": "b03476ffb1d2b7d1e01e255c73566311a0972134c2ba3f4eed5e8827756d3573", "bytes": 5015, "modes": { "ar-kv-cache": [ { "batch": 1, "first_call_ms": 99.63850025087595, "p50_ms": 81.84097870253026, "p95_ms": 83.41053728945553, "mean_ms": 82.09857293404639, "stdev_ms": 1.1577405674387165, "sequences_per_s": 12.218817710315127, "peak_memory_mb": 62.569783210754395, "raw_ms": [ 86.05066733434796, 82.56004238501191, 82.52991596236825, 81.92854234948754, 82.33833312988281, 82.98841584473848, 81.6858340986073, 82.7622078359127, 81.03462494909763, 80.7295423001051, 81.13400032743812, 81.66966680437326, 81.36783400550485, 80.9295829385519, 81.99733309447765, 83.27158307656646, 81.73391688615084, 81.57745795324445, 81.75349980592728, 81.92845759913325 ], "gpu_before": [ 0 ], "gpu_after": [ 91 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ar/ar-kv-cache-batch1.npy", "first_output_sha256": "cbd6c3c546120755bf14e1489f2fe33ddc1462a17520e15c48f2ea05cb49316b" }, { "batch": 16, "first_call_ms": 150.44429106637836, "p50_ms": 142.71245850250125, "p95_ms": 144.58474130369723, "mean_ms": 142.93104989919811, "stdev_ms": 0.8432034914764466, "sequences_per_s": 112.11354753390067, "peak_memory_mb": 201.69926834106445, "raw_ms": [ 143.9499999396503, 144.057875033468, 143.20870768278837, 142.41391699761152, 142.03700004145503, 142.45345909148455, 141.98295818641782, 142.22249994054437, 141.9028751552105, 143.3542910963297, 142.82404212281108, 142.20991590991616, 142.2273749485612, 144.6020002476871, 144.58383293822408, 143.2083328254521, 143.07395787909627, 143.31341721117496, 142.3936658538878, 142.60087488219142 ], "gpu_before": [ 0 ], "gpu_after": [ 0 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ar/ar-kv-cache-batch16.npy", "first_output_sha256": "a45396d9e1e7846ee730d57c8f46360939d43545d2c02f7581a6d2c4a1b43444" }, { "batch": 64, "first_call_ms": 348.4531659632921, "p50_ms": 357.4109794571996, "p95_ms": 365.64693509135395, "mean_ms": 358.41047710273415, "stdev_ms": 4.716975709604719, "sequences_per_s": 179.06556787146513, "peak_memory_mb": 642.0428695678711, "raw_ms": [ 351.05604119598866, 355.1087495870888, 363.3729577995837, 354.7205408103764, 353.326250333339, 354.2366670444608, 353.9806669577956, 355.22720869630575, 354.8472090624273, 355.5125412531197, 359.0411669574678, 358.66666678339243, 363.0329589359462, 356.1552921310067, 359.12891663610935, 365.4963746666908, 368.50758315995336, 361.4007500000298, 362.4345827847719, 362.9564172588289 ], "gpu_before": [ 0 ], "gpu_after": [ 80 ], "first_output_file": "runs/tinystories-e21-aligned-bench-ar/ar-kv-cache-batch64.npy", "first_output_sha256": "f0cd3865ee8580331dd485cdbba04bf09ffcf193bddc70b484376fab5430b264" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison.", "all_cold_outputs_equal_eager": null, "cold_output_consistency": null, "source": "frozen training AR sampler", "checkpoint_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } } } }, "verified": { "all_sample_ids_lengths_vocabulary_and_text_hashes": true, "sample_files_rehashed": true, "checkpoint_config_tokenizer_and_report_hashes_verified": true, "scorer_nll_aggregation_verified": true, "ordered_test_windows_rehashed": true }, "original_report_audit": { "passed": true, "samples": 5120, "scored_rows": 5120, "report_sha256": "52213efdc9fc2a550b875c0e99379a341b7caf1510dbd72cd15d943e5bcbbb14", "audit_source_sha256": "3427dacfbd745eb9f860d08f0e689c8d5add261633c968662bce25afd5fc6cbe" } }, "e22": { "name": "tinystories-e22", "normalized_config": { "name": "tinystories-e22", "out_dir": "artifacts/tinystories-e22", "data": { "seq_len": 128, "raw_dir": "artifacts/tinystories/raw", "cache_root": "artifacts/tinystories", "bpe": "artifacts/official/tinystories/mlx/bpe.json", "limit": 0, "val_rows": 2048, "unigram_rows": 200000 }, "codec": { "dim": 256, "n_head": 4, "levels": [ 1, 8, 16, 24, 32, 40, 52, 64, 76, 92, 108, 128 ], "codebook_sizes": [ 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048 ], "l0_strides": [ 4, 4, 4, 2 ], "group_norm_groups": 8, "phi_ratio": 0.5, "dropout": 0.3, "decay": 0.95, "epsilon": 1e-05, "commitment": 0.25, "fine_drop_prob": 0.5, "fine_drop_min_keep": 1, "restart_threshold": 0.01 }, "generator": { "n_layer": 6, "n_head": 6, "dim": 384, "rope_base": 100000.0, "dropout": 0.1, "corrupt_first": 0.3, "corrupt_last": 0.05, "loss_alpha": 0.0 }, "train": { "seed": 0, "log_every": 200, "eval_every": 1000, "compile": true, "codec": { "steps": 14000, "batch_size": 256, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 1000, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.01, "grad_clip": 1.0, "max_minutes": 120 }, "generator": { "steps": 15000, "batch_size": 64, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 390 }, "ar": { "steps": 7500, "batch_size": 128, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 90 } } }, "config_file_sha256": "5fe23ef7f09d9e73fd0b82217c79f74baa7a00d4796467d0bf0fdc0b852da194", "report": { "logical_path": "runs/tinystories-e22-aligned-eval/report.json", "sha256": "6b2effa71db1a8431b5884a83cbbe9ea43ba7d680c3310e67836aa8982c18fe4", "bytes": 19068, "started_local": "2026-10-30 09:36:22", "finished_local": "2026-10-30 09:38:01", "wall_seconds": 101.32440328598022 }, "runtime": { "mlx": "0.32.3", "device": "Device(gpu, 0)", "metal": true, "cuda": false, "machine": "arm64", "macos": "26.5", "python": "3.12.14", "chip": "Apple M5 Pro", "memory_gb": 64.0 }, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "identities": { "ordered_test_windows_sha256": "ae33fdb4221503283c5853f7639c163060a65d22a68f174883fb7d84baf2b66c", "data_manifest_sha256": "c576dbb4a51a290abb17532a516564a8bfd4ec5e5d61e2bb3a6da1888063490f", "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "99de25c0511d779c7d0795dda422a3b1b937191c08746331671869835017a60e", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "d1626d2f081a14f092ab495c2a118e7de0b09fdade808fa03b9fbb657dc6f556", "bytes": 57671 } }, "ar_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } }, "official_files": {}, "scorer_files": { "model.safetensors": { "sha256": "5f47f3e12f91cd33b662ce7e433b6150ad5512b5884a2cee961b50e9c3bbebce", "bytes": 3247159078 }, "config.json": { "sha256": "7fccdcfd6622055342a734c663ee0b61ff4fd697f42467595df0bf4448c8c170", "bytes": 666 }, "tokenizer.json": { "sha256": "8414cab924d8b9b33013f0d221c5862f365ee9be39c5c2bfae8a5a9e970478a6", "bytes": 1355256 } }, "job_driver_sha256": "3afe1757667ca196cf6b7d33fc27470d6c74323c19850c9590e4c482e49fe71f", "plan_sha256": "ecc669dbc593113b3143e9114abc5a37973a53da26080724fa47bd4e2d838fc2" }, "training": { "codec": { "steps": 14000, "planned_steps": 14000, "time_limited": false, "train_seconds": 5517.25, "sequences_seen": 3584000, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "val": { "sequences": 2048, "token_acc": 0.99819, "nonpad_token_acc": 0.99819, "exact_window_pct": 84.814, "recon_ce": 0.00954, "cumulative_nonpad_acc": [ 0.1325, 0.1716, 0.2614, 0.3757, 0.5327, 0.6336, 0.7484, 0.9008, 0.9377, 0.9609, 0.9756, 0.9982 ], "code_usage": [ 0.5918, 0.6001, 0.9204, 0.9805, 0.9873, 0.957, 0.9155, 0.981, 0.8838, 0.8584, 0.8848, 0.9961 ] }, "level0_codes_used": 1987, "reused_from": "tinystories-e21", "reused_frozen": true, "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145" }, "generator": { "steps": 15000, "planned_steps": 15000, "time_limited": false, "train_seconds": 14435.59, "sequences_seen": 960000, "corruption_rates": [ 0.3, 0.27499999999999997, 0.25, 0.22499999999999998, 0.19999999999999998, 0.175, 0.15, 0.125, 0.09999999999999998, 0.07499999999999998, 0.04999999999999999 ], "val": { "code_ce_per_level": [ 6.549, 4.7595, 3.902, 3.2386, 2.7903, 1.9733, 0.8392, 1.4273, 1.6172, 1.6297, 0.3215 ], "code_ce_position_weighted": 1.66904 } }, "ar_common_control": { "steps": 7500, "planned_steps": 7500, "time_limited": false, "train_seconds": 2062.03, "sequences_seen": 960000, "parameters": 13794048 } }, "models": { "ncm": { "codec_test": { "sequences": 4096, "token_acc": 0.99816, "nonpad_token_acc": 0.99816, "exact_window_pct": 84.937, "recon_ce": 0.00992, "cumulative_nonpad_acc": [ 0.1339, 0.1718, 0.2629, 0.377, 0.5347, 0.6351, 0.7495, 0.9017, 0.9376, 0.9614, 0.9764, 0.9982 ], "code_usage": [ 0.813, 0.6187, 0.9307, 0.9844, 0.9951, 0.9858, 0.9609, 0.9912, 0.9336, 0.9238, 0.9316, 0.999 ] }, "generator_test_code_ce": { "code_ce_per_level": [ 6.5527, 4.7406, 3.8811, 3.2417, 2.7699, 1.985, 0.8352, 1.4233, 1.6162, 1.625, 0.3181 ], "code_ce_position_weighted": 1.66515 }, "parameters": { "codec": 15508096, "generator": 11832587 }, "model_forwards_per_sample": 11, "diagnostics": { "samples": 1280, "mean_tokens": 127.2, "mean_chars": 505.9, "eos_in_window_pct": 8.05, "empty_pct": 0.0, "unigram_entropy": 3.9703, "duplicate_samples_pct": 0.0, "distinct_1": 0.0217, "distinct_2": 0.2312, "distinct_3": 0.5374, "repeated_4gram_pct": 0.4, "copied_train_opening_32_pct": 0.0 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 57.06, "0": 55.003, "1": 55.205, "2": 54.626, "3": 52.721 }, "mean": 54.923, "sd": 1.548, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } }, "ar": { "ar_test": { "token_ce": 1.31454, "nonpad_token_ce": 1.32042 }, "parameters": { "ar": 13794048 }, "model_forwards_per_sample": 128, "diagnostics": { "samples": 1280, "mean_tokens": 127.61, "mean_chars": 512.9, "eos_in_window_pct": 3.12, "empty_pct": 0.0, "unigram_entropy": 3.9854, "duplicate_samples_pct": 0.0, "distinct_1": 0.0251, "distinct_2": 0.2184, "distinct_3": 0.491, "repeated_4gram_pct": 0.84, "copied_train_opening_32_pct": 1.25 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 14.885, "0": 14.581, "1": 14.032, "2": 15.258, "3": 14.623 }, "mean": 14.676, "sd": 0.45, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } } }, "scorer_records": { "sha256": "4996b7d365cfb577e4c93e7b89862742e0c0e6ca4b3b8196861441ef5c4b0275", "bytes": 537842 }, "samples": { "ncm": { "sha256": "5027187700cb2922bcd356e76c0ac576c91b31b35cc8294a0db3548de67185dd", "bytes": 1553714 }, "ar": { "sha256": "718aeec386f0da4ab75f580c1bc9ba1d5eee1ce6b934f545c709047a7ebc4890", "bytes": 1565298 } }, "first_eight_ncm_samples_seed_42": [ { "model": "ncm", "seed": 42, "index": 0, "text": "\n\n He and his mum were walking by the beach beach. T Tise was very happy because he wanted to learn how to try it. He wanted to show his plane plane and take off his boat. He showed his son how sailor how he could go.\nThe day, a little girl named Sue came to said, \"Why do you do really upset What have to see it?\" Mom smiled and said, \"Yes, you too small, boat. I'm promise to drink you. Thank for special today. But I have it.\" \nSally tried to swim away He, but he was free. He was now", "ids": [ 12, 12, 319, 268, 345, 1673, 412, 1286, 862, 264, 1710, 1710, 48, 272, 272, 685, 284, 386, 384, 720, 287, 415, 267, 622, 772, 267, 821, 312, 48, 319, 415, 267, 712, 345, 2028, 2028, 268, 762, 944, 345, 990, 48, 319, 1005, 345, 4216, 772, 3928, 772, 287, 475, 489, 48, 12, 418, 330, 46, 260, 398, 486, 409, 517, 559, 267, 327, 46, 320, 1315, 382, 351, 382, 1696, 2088, 1616, 508, 267, 547, 312, 485, 730, 571, 268, 327, 46, 320, 744, 46, 351, 430, 568, 46, 990, 48, 341, 929, 2051, 267, 1782, 351, 48, 2052, 395, 857, 1718, 48, 548, 341, 508, 312, 403, 34, 34, 12, 2501, 661, 267, 1376, 581, 319, 46, 416, 287, 284, 1752, 48, 319, 284, 963 ] }, { "model": "ncm", "seed": 42, "index": 1, "text": "\nOnce upon a time, there was a big, friendly dog named Max. Max was a very reliable reliable friend who liked to. One day, Max wanted to play with the ball by the mud playing in home.\nMax rushed to run find things alone in the woods. He found a big, red ball. went back to look things he saw a Luna. Luna said, \"Hello, you have a a gift to our plan!\"!\" Max played with the magic ball ball. But then, something unexpected happened. Max went to ball with the ball and said, \"I want to provide some food for me me.\" Ben be", "ids": [ 12, 438, 447, 260, 404, 46, 408, 284, 260, 348, 46, 1347, 474, 409, 523, 48, 523, 284, 260, 386, 3191, 3191, 359, 667, 537, 267, 48, 536, 330, 46, 523, 415, 267, 328, 331, 264, 484, 862, 264, 1770, 592, 316, 591, 48, 12, 1300, 4541, 267, 745, 579, 633, 1568, 316, 264, 1494, 48, 319, 514, 260, 348, 46, 678, 484, 48, 435, 521, 267, 709, 633, 287, 390, 260, 2291, 48, 2291, 327, 46, 320, 1188, 46, 351, 508, 260, 260, 1802, 267, 1150, 1374, 421, 421, 523, 482, 331, 264, 945, 484, 484, 48, 548, 642, 46, 586, 920, 808, 48, 523, 435, 267, 484, 331, 264, 484, 268, 327, 46, 320, 75, 372, 267, 4648, 609, 811, 395, 528, 528, 403, 560, 325 ] }, { "model": "ncm", "seed": 42, "index": 2, "text": "\nOnce upon a time, there was a little girl named Lily. She loved to play hide and hide in her yard. One day, she found a new trumpet trumpet. It was a small kitten and Lily thought it was a nice toy.\nLily thought of a plan. She thought of a plan plan in the hole. She thought she helped the house and then snack. She played seek, and had lots of fun. She had been to wait to wake down.\n, something unexpected happened.As Lily Lily blew the button, shut the door. The hesitated pushed on there was inside. Lily tried to open the box on on", "ids": [ 12, 438, 447, 260, 404, 46, 408, 284, 260, 398, 486, 409, 413, 48, 342, 511, 267, 328, 1424, 268, 1424, 316, 338, 1253, 48, 536, 330, 46, 399, 514, 260, 552, 3313, 3313, 48, 432, 284, 260, 568, 2093, 268, 413, 589, 312, 284, 260, 740, 453, 48, 12, 674, 589, 373, 260, 1374, 48, 342, 589, 373, 260, 1374, 1374, 316, 264, 1090, 48, 342, 589, 399, 1003, 264, 612, 268, 642, 1744, 48, 342, 482, 2128, 46, 268, 367, 860, 373, 451, 48, 342, 367, 1414, 267, 1156, 267, 2806, 613, 48, 12, 46, 586, 920, 808, 48, 925, 413, 413, 1552, 264, 1693, 46, 3631, 264, 911, 48, 286, 6753, 1420, 355, 408, 284, 752, 48, 413, 661, 267, 1040, 264, 626, 355, 355 ] }, { "model": "ncm", "seed": 42, "index": 3, "text": " them and lady a,, in a small little girl there Lily a Lily loved to play outside outside. She had a lot of of fun. Every day, she put one one on her and went to so fun.\nOne day, a little girl named Lily was a regular sneaky. Daisy was playing in the Lily started to say, \"Lily,Hello two \" sheep, are are okay?\" Lily said, \"I need need to jump and jump? I want to feel and hurt you. Are you play with me?\" Lily Lily nodded and gave Daisy a big hug. She gave a hug and Lily helped her feel better.\n", "ids": [ 499, 268, 1305, 260, 46, 46, 316, 260, 568, 398, 486, 408, 413, 260, 413, 511, 267, 328, 707, 707, 48, 342, 367, 260, 658, 373, 373, 451, 48, 901, 330, 46, 399, 583, 735, 735, 355, 338, 268, 435, 267, 407, 451, 48, 12, 391, 330, 46, 260, 398, 486, 409, 413, 284, 260, 3350, 5693, 48, 1614, 284, 592, 316, 264, 413, 557, 267, 572, 46, 320, 674, 46, 1188, 1071, 320, 2275, 46, 492, 492, 1220, 485, 413, 327, 46, 320, 75, 778, 778, 267, 725, 268, 725, 65, 341, 372, 267, 742, 268, 907, 351, 48, 4220, 351, 328, 331, 528, 485, 413, 413, 1110, 268, 766, 1614, 260, 348, 692, 48, 342, 766, 260, 692, 268, 413, 1003, 338, 742, 831, 48, 12 ] }, { "model": "ncm", "seed": 42, "index": 4, "text": "\n\nOnce upon a time in a small village, there lived a powerful lion named Ella. Ella lived in a tree the middle of the jungle. She loved to smile who would make and laugh and giggle. Every day they played scared of the jungle and had lots of fun. One day, the bird heard Ella a old old old tree and the block b and’et by herself. She walked around and said, \"I'm being the jungle jungle there - Ella was spent the beautiful beautiful. When he played there, she replied suddenly appeared. She come down down and be scared him.\nThe bird called Ella for help he", "ids": [ 12, 12, 438, 447, 260, 404, 316, 260, 568, 1991, 46, 408, 640, 260, 2547, 1291, 409, 2885, 48, 2885, 640, 316, 260, 506, 264, 3083, 373, 264, 3005, 48, 342, 511, 267, 957, 667, 538, 539, 268, 1320, 268, 4643, 48, 901, 330, 364, 482, 617, 373, 264, 3005, 268, 367, 860, 373, 451, 48, 34, 391, 330, 46, 264, 426, 834, 2885, 260, 788, 788, 788, 506, 268, 264, 1268, 266, 268, 1323, 310, 862, 1938, 48, 342, 796, 632, 268, 327, 46, 320, 75, 929, 1019, 264, 3005, 3005, 408, 1864, 2885, 284, 2795, 264, 1000, 1000, 48, 942, 287, 482, 408, 46, 399, 1475, 2246, 2544, 48, 342, 908, 613, 613, 268, 325, 617, 483, 48, 12, 418, 426, 1027, 2885, 395, 441, 287 ] }, { "model": "ncm", "seed": 42, "index": 5, "text": "\nOne upon a time little girl named Lily went for a walk. She saw a wild mosquito. They were happy to have and have fun together.\nLily saw the bird singing. She fell down and said, \"Don't worry, bird!\" The bird decided to sleep for saw Lily and said, \"I have the birds birds first.\" The bird felt scared and felt sad.\nLily looked sad, but the first bird met a friend. Her friend the new law, and Lily said, \"It is for the wolf for the bugs.\"\n", "ids": [ 12, 391, 447, 260, 404, 398, 486, 409, 413, 435, 395, 260, 962, 48, 342, 390, 260, 2147, 3712, 48, 315, 412, 384, 267, 508, 268, 508, 451, 466, 48, 12, 674, 390, 264, 426, 1929, 48, 342, 1068, 613, 268, 327, 46, 320, 1248, 501, 1282, 46, 426, 421, 286, 426, 718, 267, 1239, 395, 390, 413, 268, 327, 46, 320, 75, 508, 264, 1116, 1116, 1182, 403, 286, 426, 543, 617, 268, 543, 503, 48, 12, 674, 512, 503, 46, 416, 264, 1182, 426, 1028, 260, 359, 48, 940, 359, 264, 552, 2527, 46, 268, 413, 327, 46, 320, 1022, 439, 395, 264, 2358, 395, 264, 2042, 403, 12, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] }, { "model": "ncm", "seed": 42, "index": 6, "text": "\nOne sunny day, Sue Sue was playing in the park with her mom. She saw a big apple on the table. She wanted to roll it to spoil and the taste smell good. She quickly ran to the kitchen and ran over. She asked mom, \"Mom, I carry the apple apple?\"\nHer mom smiled and said, \"Yes, it's easy to drop make an apple.amamam. \"What is good for us?\" asked Mr. Brown. \nSue didn't listen to Mr, but scared got mad. Sue got and told him a kiss. She gave Bear a big hug and gave her to", "ids": [ 12, 391, 1167, 330, 46, 517, 517, 284, 592, 316, 264, 530, 331, 338, 394, 48, 342, 390, 260, 348, 1454, 355, 264, 1280, 48, 342, 415, 267, 1326, 312, 267, 4450, 268, 264, 2225, 1388, 567, 48, 342, 1545, 602, 267, 264, 1173, 268, 602, 886, 48, 342, 550, 394, 46, 320, 848, 46, 341, 2026, 264, 1454, 1454, 485, 12, 1585, 394, 571, 268, 327, 46, 320, 744, 46, 312, 392, 2383, 267, 2895, 539, 428, 1454, 48, 309, 309, 309, 48, 320, 1076, 439, 567, 395, 829, 485, 550, 1325, 48, 2957, 48, 34, 12, 1341, 813, 501, 994, 267, 1325, 46, 416, 617, 665, 1880, 48, 517, 665, 268, 747, 483, 260, 2065, 48, 342, 766, 2664, 260, 348, 692, 268, 766, 338, 267 ] }, { "model": "ncm", "seed": 42, "index": 7, "text": " theOne room a time, was was a charming dog named Spot. Spot loved to run and play his friends. Spot always liked play and hide in the sun all long.\nOne day, a little bird named Fluffy came to visit visit Spot. \"Let's play the tent and play friends!\" said Spot. Spot said, \"Yes, Spot! I Mr! S music! Can I be friends friends?\" Spot barked barked and barked, \"Yes, please too!\" They, one day, Spot and Whiskers Blue played together and became friends. They played in the game all day, and had have fun.\n", "ids": [ 264, 391, 716, 260, 404, 46, 284, 284, 260, 3408, 474, 409, 601, 48, 601, 511, 267, 745, 268, 328, 345, 419, 48, 601, 710, 537, 328, 268, 1424, 316, 264, 643, 440, 746, 48, 12, 391, 330, 46, 260, 398, 426, 409, 1130, 559, 267, 1708, 1708, 601, 48, 320, 904, 392, 328, 264, 2661, 268, 328, 419, 421, 327, 601, 48, 601, 327, 46, 320, 744, 46, 601, 35, 341, 1325, 35, 299, 1626, 35, 1129, 341, 325, 419, 419, 485, 601, 1813, 1813, 268, 1813, 46, 320, 744, 46, 1294, 430, 421, 315, 46, 735, 330, 46, 601, 268, 2015, 2029, 482, 466, 268, 698, 419, 48, 315, 482, 316, 264, 1031, 440, 330, 46, 268, 367, 508, 451, 48, 12, 0, 1, 1, 1 ] } ], "recomputed_ppl": { "ncm": { "per_seed": { "42": 57.05972532546591, "0": 55.003193534618774, "1": 55.2046390019606, "2": 54.62644116929124, "3": 52.72131792554102 }, "mean": 54.92306339137551, "sd_sampling_seed": 1.5481196436225808, "max_scored_tokens": 144, "unscorable": 0 }, "ar": { "per_seed": { "42": 14.885483249080956, "0": 14.580821038376598, "1": 14.031941107377678, "2": 15.25808914998773, "3": 14.623375352318366 }, "mean": 14.675941979428266, "sd_sampling_seed": 0.4498197524543124, "max_scored_tokens": 152, "unscorable": 0 } }, "benchmarks": { "ncm": { "file": "tinystories-e22-aligned-bench-ncm.json", "sha256": "f15a7e1260f1e27e20f0ae5a4308c99dae19fb914c6389b72ac0e31b56953d94", "bytes": 14081, "modes": { "eager": [ { "batch": 1, "first_call_ms": 31.947833951562643, "p50_ms": 13.269541319459677, "p95_ms": 14.056886010803282, "mean_ms": 13.381295907311141, "stdev_ms": 0.364008577560238, "sequences_per_s": 75.3605551183226, "peak_memory_mb": 184.9329376220703, "raw_ms": [ 14.05020896345377, 13.922582846134901, 13.842083048075438, 14.183749910444021, 13.193249702453613, 13.303082901984453, 13.632667250931263, 13.293582946062088, 12.890417128801346, 13.17062508314848, 13.288124930113554, 13.443958014249802, 13.123792130500078, 13.20479204878211, 13.194750063121319, 12.909084092825651, 13.09416675940156, 13.471292331814766, 13.2509577088058, 13.162750285118818 ], "gpu_before": [ 0 ], "gpu_after": [ 84 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/eager-batch1.npy", "first_output_sha256": "23d5aa6ef17ac1dcdabde3b6dfbb054bbfd9de505568c08d0e1677a36ada1299" }, { "batch": 16, "first_call_ms": 58.058416936546564, "p50_ms": 51.63791659288108, "p95_ms": 59.40187887754291, "mean_ms": 52.1250061923638, "stdev_ms": 3.3239810656083164, "sequences_per_s": 309.84983623847046, "peak_memory_mb": 1114.4673776626587, "raw_ms": [ 48.44695841893554, 48.49774995818734, 52.40091681480408, 53.45612484961748, 50.30512483790517, 52.10858304053545, 52.682249806821346, 50.81841675564647, 52.282500080764294, 51.88666610047221, 51.634958013892174, 50.72266701608896, 50.16641691327095, 51.36245768517256, 49.406083300709724, 51.64087517186999, 50.77604204416275, 52.08799988031387, 59.23445802181959, 62.58287513628602 ], "gpu_before": [ 0 ], "gpu_after": [ 80 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/eager-batch16.npy", "first_output_sha256": "e948008a5ac90c1b4626c8b3c97329b9d858b6954cf428fdba2442216f9552ee" }, { "batch": 64, "first_call_ms": 323.7671251408756, "p50_ms": 275.51591699011624, "p95_ms": 331.92474809475243, "mean_ms": 283.0052271252498, "stdev_ms": 26.101314895751486, "sequences_per_s": 232.29147956012977, "peak_memory_mb": 2256.574296951294, "raw_ms": [ 266.7146250605583, 261.8518332019448, 235.86941696703434, 249.1224161349237, 274.4616251438856, 274.5189592242241, 260.9971249476075, 266.0674159415066, 276.5128747560084, 265.836332924664, 265.689667314291, 291.09416669234633, 298.84745879098773, 292.1126252040267, 301.9458749331534, 336.1807088367641, 311.0151248984039, 301.2139583006501, 331.70075016096234, 298.351583071053 ], "gpu_before": [ 7 ], "gpu_after": [ 88 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/eager-batch64.npy", "first_output_sha256": "25b6b99d4ac9512d4f68726f09eac868a501d2405d9942cbe59b66ff40b6fa45" } ], "cached": [ { "batch": 1, "first_call_ms": 22.176458965986967, "p50_ms": 14.784812927246094, "p95_ms": 15.147725050337613, "mean_ms": 14.813801972195506, "stdev_ms": 0.24547970967523197, "sequences_per_s": 67.63697348900213, "peak_memory_mb": 183.0602617263794, "raw_ms": [ 14.886667020618916, 15.11400006711483, 15.384749975055456, 14.681042172014713, 15.033041127026081, 14.543582685291767, 15.135250054299831, 14.827500097453594, 15.027874615043402, 14.751916751265526, 14.740082900971174, 14.98166611418128, 14.502249658107758, 14.660874847322702, 14.549915678799152, 14.672541990876198, 14.928124845027924, 14.536708127707243, 14.817709103226662, 14.500541612505913 ], "gpu_before": [ 0 ], "gpu_after": [ 56 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached-batch1.npy", "first_output_sha256": "23d5aa6ef17ac1dcdabde3b6dfbb054bbfd9de505568c08d0e1677a36ada1299" }, { "batch": 16, "first_call_ms": 92.96508273109794, "p50_ms": 72.50785431824625, "p95_ms": 126.76215195097036, "mean_ms": 85.65393132157624, "stdev_ms": 40.890989615220995, "sequences_per_s": 220.66574925488698, "peak_memory_mb": 1112.5691890716553, "raw_ms": [ 76.84774976223707, 78.94050003960729, 68.16795887425542, 66.65404187515378, 63.32933297380805, 61.82274967432022, 62.86379089578986, 54.76187542080879, 54.99975010752678, 52.833832800388336, 53.13379177823663, 55.900041945278645, 86.43991686403751, 90.10895807296038, 105.64849991351366, 121.29449984058738, 109.03345933184028, 104.74799992516637, 114.90233428776264, 230.64754204824567 ], "gpu_before": [ 0 ], "gpu_after": [ 92 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached-batch16.npy", "first_output_sha256": "e948008a5ac90c1b4626c8b3c97329b9d858b6954cf428fdba2442216f9552ee" }, { "batch": 64, "first_call_ms": 579.7657500952482, "p50_ms": 334.61275044828653, "p95_ms": 366.80260605644435, "mean_ms": 334.516135440208, "stdev_ms": 17.435307868256412, "sequences_per_s": 191.2658735037983, "peak_memory_mb": 2413.095678329468, "raw_ms": [ 322.01487524434924, 320.22391725331545, 309.78420935571194, 338.1822910159826, 354.7339579090476, 337.5388327986002, 321.9246659427881, 319.27112489938736, 326.5414577908814, 338.08333333581686, 325.58970898389816, 313.7519173324108, 328.40554183349013, 335.79683396965265, 337.055291980505, 339.58191564306617, 339.93454184383154, 333.4286669269204, 365.97624979913235, 382.50337494537234 ], "gpu_before": [ 100 ], "gpu_after": [ 72 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached-batch64.npy", "first_output_sha256": "25b6b99d4ac9512d4f68726f09eac868a501d2405d9942cbe59b66ff40b6fa45" } ], "cached+compiled": [ { "batch": 1, "first_call_ms": 2626.060000155121, "p50_ms": 12.41289614699781, "p95_ms": 12.998318625614047, "mean_ms": 12.504266807809472, "stdev_ms": 0.3071601366501558, "sequences_per_s": 80.56137650373081, "peak_memory_mb": 183.40471076965332, "raw_ms": [ 13.123915996402502, 12.87412503734231, 12.713209260255098, 12.608374934643507, 12.476041913032532, 12.586125172674656, 12.288582976907492, 12.30958430096507, 12.796500232070684, 12.101833708584309, 12.061875313520432, 12.293041683733463, 12.812250293791294, 12.349750380963087, 12.991708237677813, 12.313666753470898, 12.666291557252407, 12.164166197180748, 12.278125155717134, 12.276167050004005 ], "gpu_before": [ 0 ], "gpu_after": [ 71 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached+compiled-batch1.npy", "first_output_sha256": "23d5aa6ef17ac1dcdabde3b6dfbb054bbfd9de505568c08d0e1677a36ada1299" }, { "batch": 16, "first_call_ms": 64.68745786696672, "p50_ms": 45.7185001578182, "p95_ms": 49.98465576209128, "mean_ms": 46.25898147933185, "stdev_ms": 1.9982472189272855, "sequences_per_s": 349.9677361411403, "peak_memory_mb": 1150.3402767181396, "raw_ms": [ 50.7294163107872, 49.64820807799697, 49.945457838475704, 48.17558405920863, 47.99629235640168, 45.906833838671446, 46.35208332911134, 45.55229190737009, 45.7820831798017, 46.25079222023487, 45.22145912051201, 46.37208301573992, 44.27970899268985, 44.36975019052625, 44.782625045627356, 45.01362517476082, 44.4374168291688, 45.654917135834694, 44.28270924836397, 44.42629171535373 ], "gpu_before": [ 0 ], "gpu_after": [ 93 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached+compiled-batch16.npy", "first_output_sha256": "e948008a5ac90c1b4626c8b3c97329b9d858b6954cf428fdba2442216f9552ee" }, { "batch": 64, "first_call_ms": 314.30941680446267, "p50_ms": 321.40568690374494, "p95_ms": 446.79052704013884, "mean_ms": 330.95425623469055, "stdev_ms": 67.15683322915467, "sequences_per_s": 199.1252880947524, "peak_memory_mb": 2551.9475450515747, "raw_ms": [ 253.38224973529577, 240.6047498807311, 235.52558291703463, 241.72191601246595, 267.96912495046854, 326.02541567757726, 384.4573749229312, 446.48312497884035, 311.8358748033643, 331.4017918892205, 375.0975839793682, 388.93837481737137, 357.58558427914977, 397.09591679275036, 452.63116620481014, 401.0717920027673, 316.7859581299126, 290.9025833941996, 302.0397094078362, 297.5292499177158 ], "gpu_before": [ 0 ], "gpu_after": [ 79 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ncm/cached+compiled-batch64.npy", "first_output_sha256": "25b6b99d4ac9512d4f68726f09eac868a501d2405d9942cbe59b66ff40b6fa45" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison.", "all_cold_outputs_equal_eager": true, "cold_output_consistency": [ { "mode": "eager", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 16, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 64, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 16, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 64, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached+compiled", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached+compiled", "batch": 16, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached+compiled", "batch": 64, "tokens_equal": true, "different_token_fraction": 0.0 } ], "source": "frozen optimized MLX sampler", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "99de25c0511d779c7d0795dda422a3b1b937191c08746331671869835017a60e", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "d1626d2f081a14f092ab495c2a118e7de0b09fdade808fa03b9fbb657dc6f556", "bytes": 57671 } } }, "ar": { "file": "tinystories-e22-aligned-bench-ar.json", "sha256": "70a86a87c8ed1f3a90ab3d2febbc6a6708fe2916be99e29af65e9d98c5a708d3", "bytes": 5022, "modes": { "ar-kv-cache": [ { "batch": 1, "first_call_ms": 130.4877088405192, "p50_ms": 88.99749931879342, "p95_ms": 107.86593558732422, "mean_ms": 93.32155815791339, "stdev_ms": 8.812134460204387, "sequences_per_s": 11.23627076776563, "peak_memory_mb": 62.569783210754395, "raw_ms": [ 96.04645799845457, 96.1267501115799, 89.18950008228421, 89.03216570615768, 87.35999977216125, 86.93616604432464, 88.96283293142915, 91.25416725873947, 88.0320412106812, 88.23875011876225, 85.67800000309944, 85.91841673478484, 87.69920794293284, 86.76458289846778, 88.277124799788, 118.55379212647676, 107.30341682210565, 101.24024981632829, 104.70412485301495, 99.11341592669487 ], "gpu_before": [ 0 ], "gpu_after": [ 74 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ar/ar-kv-cache-batch1.npy", "first_output_sha256": "cbd6c3c546120755bf14e1489f2fe33ddc1462a17520e15c48f2ea05cb49316b" }, { "batch": 16, "first_call_ms": 214.36658268794417, "p50_ms": 148.98247900418937, "p95_ms": 151.73405359964818, "mean_ms": 148.8035603892058, "stdev_ms": 2.5861198183129646, "sequences_per_s": 107.39517899651865, "peak_memory_mb": 202.26961517333984, "raw_ms": [ 153.92325026914477, 149.8595830053091, 146.1988748051226, 144.81670828536153, 145.3287503682077, 149.37058370560408, 148.95216701552272, 147.61729212477803, 145.91741701588035, 149.01033276692033, 143.64216709509492, 148.95462524145842, 151.46783273667097, 148.68179196491838, 150.75512509793043, 148.8638329319656, 149.4191661477089, 150.24033328518271, 151.61883272230625, 151.4325411990285 ], "gpu_before": [ 29 ], "gpu_after": [ 87 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ar/ar-kv-cache-batch16.npy", "first_output_sha256": "a45396d9e1e7846ee730d57c8f46360939d43545d2c02f7581a6d2c4a1b43444" }, { "batch": 64, "first_call_ms": 409.573916811496, "p50_ms": 433.7154580280185, "p95_ms": 652.8193828649819, "mean_ms": 477.5442519923672, "stdev_ms": 96.80656910954023, "sequences_per_s": 147.56218349004644, "peak_memory_mb": 642.0428695678711, "raw_ms": [ 638.8082499615848, 614.57599978894, 651.6262078657746, 675.4897078499198, 495.60529086738825, 492.27345921099186, 503.41450003907084, 532.1955420076847, 481.57400032505393, 453.80987506359816, 410.96091689541936, 391.6322919540107, 395.12204099446535, 400.37995809689164, 398.33583403378725, 413.6210409924388, 412.150832824409, 390.6652079895139, 395.75324999168515, 402.8908330947161 ], "gpu_before": [ 0 ], "gpu_after": [ 79 ], "first_output_file": "runs/tinystories-e22-aligned-bench-ar/ar-kv-cache-batch64.npy", "first_output_sha256": "f0cd3865ee8580331dd485cdbba04bf09ffcf193bddc70b484376fab5430b264" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison.", "all_cold_outputs_equal_eager": null, "cold_output_consistency": null, "source": "frozen training AR sampler", "checkpoint_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } } } }, "verified": { "all_sample_ids_lengths_vocabulary_and_text_hashes": true, "sample_files_rehashed": true, "checkpoint_config_tokenizer_and_report_hashes_verified": true, "scorer_nll_aggregation_verified": true, "ordered_test_windows_rehashed": true }, "original_report_audit": { "passed": true, "samples": 2560, "scored_rows": 2560, "report_sha256": "6b2effa71db1a8431b5884a83cbbe9ea43ba7d680c3310e67836aa8982c18fe4", "audit_source_sha256": "3427dacfbd745eb9f860d08f0e689c8d5add261633c968662bce25afd5fc6cbe" } }, "e23": { "name": "tinystories-e23", "normalized_config": { "name": "tinystories-e23", "out_dir": "artifacts/tinystories-e23", "data": { "seq_len": 128, "raw_dir": "artifacts/tinystories/raw", "cache_root": "artifacts/tinystories", "bpe": "artifacts/official/tinystories/mlx/bpe.json", "limit": 0, "val_rows": 2048, "unigram_rows": 200000 }, "codec": { "dim": 256, "n_head": 4, "levels": [ 1, 8, 16, 24, 32, 40, 52, 64, 76, 92, 108, 128 ], "codebook_sizes": [ 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048 ], "l0_strides": [ 4, 4, 4, 2 ], "group_norm_groups": 8, "phi_ratio": 0.5, "dropout": 0.3, "decay": 0.95, "epsilon": 1e-05, "commitment": 0.25, "fine_drop_prob": 0.5, "fine_drop_min_keep": 1, "restart_threshold": 0.01 }, "generator": { "n_layer": 6, "n_head": 6, "dim": 384, "rope_base": 100000.0, "dropout": 0.1, "corrupt_first": 0.0, "corrupt_last": 0.0, "loss_alpha": 0.0 }, "train": { "seed": 0, "log_every": 200, "eval_every": 1000, "compile": true, "codec": { "steps": 14000, "batch_size": 256, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 1000, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.01, "grad_clip": 1.0, "max_minutes": 120 }, "generator": { "steps": 15000, "batch_size": 64, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 390 }, "ar": { "steps": 7500, "batch_size": 128, "lr": 0.0005, "lr_floor": 5e-05, "warmup": 500, "beta1": 0.9, "beta2": 0.95, "weight_decay": 0.1, "grad_clip": 1.0, "max_minutes": 90 } } }, "config_file_sha256": "090919c48de051c2469d44d26df996e670947b1473f20c125df86d25c1c84c15", "report": { "logical_path": "runs/tinystories-e23-aligned-eval/report.json", "sha256": "2b2902e54cbefa2343da6d31e5a41ee0e2bba00d410b8d7c2485d49d2ee6d234", "bytes": 19132, "started_local": "2026-10-30 15:24:43", "finished_local": "2026-10-30 15:26:46", "wall_seconds": 124.39928603172302 }, "runtime": { "mlx": "0.32.3", "device": "Device(gpu, 0)", "metal": true, "cuda": false, "machine": "arm64", "macos": "26.5", "python": "3.12.14", "chip": "Apple M5 Pro", "memory_gb": 64.0 }, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "identities": { "ordered_test_windows_sha256": "ae33fdb4221503283c5853f7639c163060a65d22a68f174883fb7d84baf2b66c", "data_manifest_sha256": "c576dbb4a51a290abb17532a516564a8bfd4ec5e5d61e2bb3a6da1888063490f", "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "9f4db72ff15c0eae2698a79971c48f25b2fcdefa590244e87943d0b0d74933c6", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "0fed73a78ee71a37747199e5a3b492ca5e64c15e4aae124ce1dcfa5f40078498", "bytes": 57434 } }, "ar_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } }, "official_files": {}, "scorer_files": { "model.safetensors": { "sha256": "5f47f3e12f91cd33b662ce7e433b6150ad5512b5884a2cee961b50e9c3bbebce", "bytes": 3247159078 }, "config.json": { "sha256": "7fccdcfd6622055342a734c663ee0b61ff4fd697f42467595df0bf4448c8c170", "bytes": 666 }, "tokenizer.json": { "sha256": "8414cab924d8b9b33013f0d221c5862f365ee9be39c5c2bfae8a5a9e970478a6", "bytes": 1355256 } }, "job_driver_sha256": "3afe1757667ca196cf6b7d33fc27470d6c74323c19850c9590e4c482e49fe71f", "plan_sha256": "ecc669dbc593113b3143e9114abc5a37973a53da26080724fa47bd4e2d838fc2" }, "training": { "codec": { "steps": 14000, "planned_steps": 14000, "time_limited": false, "train_seconds": 5517.25, "sequences_seen": 3584000, "data_manifest": { "cache_version": 1, "dataset": "roneneldan/TinyStories", "revision": "f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", "license": "CDLA-Sharing-1.0", "files": { "TinyStoriesV2-GPT4-train.txt": "6418d412de72888f52b5142c761ac21a582f7d1166f0bfbdb5f03ccfdec90443", "TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d" }, "bpe_sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "seq_len": 128, "eos_id": 0, "pad_id": 1, "limit": 0, "heldout_overlap_removed": 0, "split": "train file -> train; valid file -> val/test by sha1(story.strip()) parity", "splits": { "train": { "stories": 2717495, "positions": 347839360, "non_pad_tokens": 346500481, "pad_fraction": 0.0038, "with_eos_in_window": 0.0498 }, "val": { "stories": 13960, "positions": 1786880, "non_pad_tokens": 1780370, "pad_fraction": 0.0036, "with_eos_in_window": 0.0496 }, "test": { "stories": 13670, "positions": 1749760, "non_pad_tokens": 1742575, "pad_fraction": 0.0041, "with_eos_in_window": 0.051 } } }, "val": { "sequences": 2048, "token_acc": 0.99819, "nonpad_token_acc": 0.99819, "exact_window_pct": 84.814, "recon_ce": 0.00954, "cumulative_nonpad_acc": [ 0.1325, 0.1716, 0.2614, 0.3757, 0.5327, 0.6336, 0.7484, 0.9008, 0.9377, 0.9609, 0.9756, 0.9982 ], "code_usage": [ 0.5918, 0.6001, 0.9204, 0.9805, 0.9873, 0.957, 0.9155, 0.981, 0.8838, 0.8584, 0.8848, 0.9961 ] }, "level0_codes_used": 1987, "reused_from": "tinystories-e21", "reused_frozen": true, "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145" }, "generator": { "steps": 15000, "planned_steps": 15000, "time_limited": false, "train_seconds": 20734.07, "sequences_seen": 960000, "corruption_rates": null, "val": { "code_ce_per_level": [ 6.5327, 4.6789, 3.8027, 3.096, 2.629, 1.7727, 0.7124, 1.0937, 1.1304, 1.0832, 0.2139 ], "code_ce_position_weighted": 1.39356 } }, "ar_common_control": { "steps": 7500, "planned_steps": 7500, "time_limited": false, "train_seconds": 2062.03, "sequences_seen": 960000, "parameters": 13794048 } }, "models": { "ncm": { "codec_test": { "sequences": 4096, "token_acc": 0.99816, "nonpad_token_acc": 0.99816, "exact_window_pct": 84.937, "recon_ce": 0.00992, "cumulative_nonpad_acc": [ 0.1339, 0.1718, 0.2629, 0.377, 0.5347, 0.6351, 0.7495, 0.9017, 0.9376, 0.9614, 0.9764, 0.9982 ], "code_usage": [ 0.813, 0.6187, 0.9307, 0.9844, 0.9951, 0.9858, 0.9609, 0.9912, 0.9336, 0.9238, 0.9316, 0.999 ] }, "generator_test_code_ce": { "code_ce_per_level": [ 6.5385, 4.6629, 3.7842, 3.1036, 2.6019, 1.7896, 0.7077, 1.0854, 1.1347, 1.0764, 0.2106 ], "code_ce_position_weighted": 1.38995 }, "parameters": { "codec": 15508096, "generator": 11832587 }, "model_forwards_per_sample": 11, "diagnostics": { "samples": 1280, "mean_tokens": 127.14, "mean_chars": 516.6, "eos_in_window_pct": 7.5, "empty_pct": 0.0, "unigram_entropy": 4.0353, "duplicate_samples_pct": 0.0, "distinct_1": 0.0263, "distinct_2": 0.2876, "distinct_3": 0.6239, "repeated_4gram_pct": 0.19, "copied_train_opening_32_pct": 0.0 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 100.459, "0": 95.578, "1": 95.674, "2": 98.571, "3": 98.599 }, "mean": 97.776, "sd": 2.107, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } }, "ar": { "ar_test": { "token_ce": 1.31454, "nonpad_token_ce": 1.32042 }, "parameters": { "ar": 13794048 }, "model_forwards_per_sample": 128, "diagnostics": { "samples": 1280, "mean_tokens": 127.61, "mean_chars": 512.9, "eos_in_window_pct": 3.12, "empty_pct": 0.0, "unigram_entropy": 3.9854, "duplicate_samples_pct": 0.0, "distinct_1": 0.0251, "distinct_2": 0.2184, "distinct_3": 0.491, "repeated_4gram_pct": 0.84, "copied_train_opening_32_pct": 1.25 }, "gen_ppl": { "scorer": "openai-community/gpt2-large@32b71b12589c2f8d625668d2335a01cac3249519", "per_seed": { "42": 14.885, "0": 14.581, "1": 14.032, "2": 15.258, "3": 14.623 }, "mean": 14.676, "sd": 0.45, "unscorable_samples": 0, "aggregation": "corpus PPL per seed; arithmetic mean across seeds" } } }, "scorer_records": { "sha256": "04cc5dd9fd21cca4dc482e2d3b17e45ea7a26fbf5c9f5f915adb81792db5824f", "bytes": 537399 }, "samples": { "ncm": { "sha256": "46de46dc6aeb7613de2a72b9e8388b45fcdbf06271d623fa2df56d7259ed0a74", "bytes": 1573403 }, "ar": { "sha256": "718aeec386f0da4ab75f580c1bc9ba1d5eee1ce6b934f545c709047a7ebc4890", "bytes": 1565298 } }, "first_eight_ncm_samples_seed_42": [ { "model": "ncm", "seed": 42, "index": 0, "text": "\n\nJack and his mum were walking by the beach at. T T gears. Tom loved to harbor because learn learned he wanted to travel beyond and was and excited, green trees and trees.\nOne day there there were in the park Tommy saw a lizard. They saw a squirrel and named Sally. Sally said, \"Hi weather, you know where the shade is we going?\" \nTim saw a big duck, and ducks catch Bob.\nBob said, in \" mild to barber. If you'll bring you come”.\nWhen when Bob got to run, the duck swam swimming away. Now his bed he knows theiling", "ids": [ 12, 12, 2000, 268, 345, 1673, 412, 1286, 862, 264, 1710, 457, 48, 272, 272, 6632, 48, 423, 511, 267, 4548, 720, 622, 694, 287, 415, 267, 3513, 7079, 268, 284, 268, 738, 46, 1069, 1284, 268, 1284, 48, 12, 391, 330, 408, 408, 412, 316, 264, 530, 1855, 390, 260, 3277, 48, 315, 390, 260, 1106, 268, 409, 1049, 48, 1049, 327, 46, 320, 1101, 4342, 46, 351, 663, 1083, 264, 3340, 439, 366, 1118, 485, 34, 12, 655, 390, 260, 348, 1213, 46, 268, 2308, 1127, 715, 48, 12, 1604, 327, 46, 316, 320, 3756, 267, 3359, 48, 2560, 351, 1973, 1914, 351, 908, 6081, 12, 1043, 638, 715, 665, 267, 745, 46, 264, 1213, 1514, 2529, 581, 48, 1155, 345, 1047, 287, 3707, 264, 2168 ] }, { "model": "ncm", "seed": 42, "index": 1, "text": "\n upon big a time, there was a big, yummy named Max. Max was strong, and strong snow in the snow. One day, by Max in Max continued to racing in the stream in swim.\nThe people wanted to bury bury the things in hide the side by the same block. He wanted to help look at his friends and he saw him there. He said, \"Hello, family for a surprise!\" Max said they played finished with his friends.\nBut then Max, something unexpected happened. He became a sword and in his mouth. Bob, and said, \"I hear thank you, I will be Ben.", "ids": [ 12, 447, 348, 260, 404, 46, 408, 284, 260, 348, 46, 1045, 409, 523, 48, 523, 284, 970, 46, 268, 970, 1430, 316, 264, 1430, 48, 536, 330, 46, 862, 523, 316, 523, 1753, 267, 4939, 316, 264, 3405, 316, 1376, 48, 12, 418, 899, 415, 267, 4171, 4171, 264, 633, 316, 1424, 264, 1675, 862, 264, 1901, 1268, 48, 319, 415, 267, 441, 709, 457, 345, 419, 268, 287, 390, 483, 408, 48, 319, 327, 46, 320, 1188, 46, 953, 395, 260, 1275, 421, 523, 327, 364, 482, 1697, 331, 345, 419, 48, 12, 780, 642, 523, 46, 586, 920, 808, 48, 319, 698, 260, 2770, 268, 316, 345, 1521, 48, 715, 46, 268, 327, 46, 320, 75, 1098, 1443, 351, 46, 341, 625, 325, 560, 48 ] }, { "model": "ncm", "seed": 42, "index": 2, "text": "\n was a a there, there was a little girl named Lily. She loved to play with her friends in the park. One day, she took her to a trumpet. It was a cute cute! Lily and thought it was good idea to bury something in the path. She thought of plan to a big tree. She run and dreamed of it and fun played together, hats and imagined. Sometimes she had a dream. In her her dream, she would sleep and asleep.\nLucy heard something in the the. He told her about to do how it was in the box. She decided decided to find the key and the top on", "ids": [ 12, 284, 260, 260, 408, 46, 408, 284, 260, 398, 486, 409, 413, 48, 342, 511, 267, 328, 331, 338, 419, 316, 264, 530, 48, 536, 330, 46, 399, 616, 338, 267, 260, 3313, 48, 432, 284, 260, 2005, 2005, 35, 413, 268, 589, 312, 284, 567, 830, 267, 4171, 586, 316, 264, 2198, 48, 342, 589, 373, 1374, 267, 260, 348, 506, 48, 342, 745, 268, 4038, 373, 312, 268, 451, 482, 466, 46, 2968, 268, 4386, 48, 2687, 399, 367, 260, 1464, 48, 965, 338, 338, 1464, 46, 399, 538, 1239, 268, 2988, 48, 12, 1592, 834, 586, 316, 264, 264, 48, 319, 747, 338, 673, 267, 382, 772, 312, 284, 316, 264, 626, 48, 342, 718, 718, 267, 579, 264, 1607, 268, 264, 1292, 355 ] }, { "model": "ncm", "seed": 42, "index": 3, "text": "\n\n\nOne day, a terrible at Mom with Dad and and Dad. They liked to watch them flowers and she gave Dad Dad a lot of fun but. He put outside hat off and on the famous man.\nOne day, a little girl named Lily. She was a young and was completely the big and white coat!\"\nHer mom said, \"Hello sheep, you are angry, Mommy Mommy. I need to pick up stage and jump and I am not hungry. Are you caught and get down with me?\"\nLily smiled and thanked the queen. She She a bite. She. kissed him and gave them a kiss", "ids": [ 12, 12, 12, 391, 330, 46, 260, 2918, 457, 730, 331, 1304, 268, 268, 1304, 48, 315, 537, 267, 1064, 499, 1074, 268, 399, 766, 1304, 1304, 260, 658, 373, 451, 416, 48, 319, 583, 707, 1026, 944, 268, 355, 264, 2725, 509, 48, 12, 391, 330, 46, 260, 398, 486, 409, 413, 48, 342, 284, 260, 2118, 268, 284, 6839, 264, 348, 268, 1610, 1910, 421, 12, 1585, 394, 327, 46, 320, 1188, 2275, 46, 351, 492, 1036, 46, 1554, 1554, 48, 341, 778, 267, 1266, 370, 3000, 268, 725, 268, 341, 749, 368, 1332, 48, 4220, 351, 1868, 268, 653, 613, 331, 528, 485, 12, 674, 571, 268, 930, 264, 2468, 48, 342, 342, 260, 1754, 48, 342, 48, 3230, 483, 268, 766, 499, 260, 2065 ] }, { "model": "ncm", "seed": 42, "index": 4, "text": "\nOnce upon a time, in a small village, there lived a lion named Leo. He was very patient tree and lived to the woods jungle. All the lion would laugh and laugh loud.\nHe liked to play play in the forest and having had lots of the little lion. One day, the little bird heard with a old tree and saw Leo's mane and leaves all by.\nAsLeo walked and walked, \"I can all pick the same like to flowers on the tree. \" live played in the with flowers,\" she said. So he climbed down down, and the hills up them. When the man was he climbed", "ids": [ 12, 438, 447, 260, 404, 46, 316, 260, 568, 1991, 46, 408, 640, 260, 1291, 409, 2240, 48, 319, 284, 386, 2079, 506, 268, 640, 267, 264, 1494, 3005, 48, 1397, 264, 1291, 538, 1320, 268, 1320, 917, 48, 12, 1033, 537, 267, 328, 328, 316, 264, 975, 268, 1120, 367, 860, 373, 264, 398, 1291, 48, 536, 330, 46, 264, 398, 426, 834, 331, 260, 788, 506, 268, 390, 2240, 392, 6044, 268, 1308, 440, 862, 48, 12, 925, 5418, 796, 268, 796, 46, 320, 75, 476, 440, 1266, 264, 1901, 525, 267, 1074, 355, 264, 506, 48, 320, 2083, 482, 316, 264, 331, 1074, 646, 399, 327, 48, 722, 287, 1367, 613, 613, 46, 268, 264, 4879, 370, 499, 48, 942, 264, 509, 284, 287, 1367 ] }, { "model": "ncm", "seed": 42, "index": 5, "text": "\n\n a a time, a girl named Sue and her dog went for a walk. They needed to cook toy and have fun to have.\n\nSpot started to wash boxes and break down. The dog growled at and started to play with the. The bird was a bird named Luna and said, \"I need like to play and play with button!\"\nThey felt sad and cried.\nSo, they went to the market. The new bird brought a new brings, and it said, \"Buddy is lived! Buddy played played in the corner, and would?\"", "ids": [ 12, 12, 260, 260, 404, 46, 260, 486, 409, 517, 268, 338, 474, 435, 395, 260, 962, 48, 315, 1313, 267, 1113, 453, 268, 508, 451, 267, 508, 48, 12, 12, 1622, 557, 267, 2301, 3427, 268, 1617, 613, 48, 286, 474, 4979, 457, 268, 557, 267, 328, 331, 264, 48, 286, 426, 284, 260, 426, 409, 2291, 268, 327, 46, 320, 75, 778, 525, 267, 328, 268, 328, 331, 1693, 421, 12, 732, 543, 503, 268, 1318, 48, 12, 1391, 46, 364, 435, 267, 264, 3002, 48, 286, 552, 426, 1669, 260, 552, 5168, 46, 268, 312, 327, 46, 320, 4731, 439, 640, 35, 1597, 482, 482, 316, 264, 2750, 46, 268, 1, 1, 1, 1, 1, 1, 1, 1, 1, 538, 1, 485, 1, 1, 1 ] }, { "model": "ncm", "seed": 42, "index": 6, "text": "\nOne day, a little girl named Lucy went outside to with her friends. She saw a big lollipop on the table and put it on it. Lucy smelled the hat and smelled good. A smell kept spaghetti. He ran over and her her mom, \"Mom, the please I yogurt yogurt too?\"\n\"Me smiled and said, \"Yes, what is. Maybe it it needs an sausages today.\" Lucy felt embarrassed. She knew the dog was too hungry, but too and she didn't like anything anything.\nJust then, Lucy got mad that that Lucy told her wider. She gave Lucy big hug and hug gave the cone", "ids": [ 12, 391, 330, 46, 260, 398, 486, 409, 693, 435, 707, 267, 331, 338, 419, 48, 342, 390, 260, 348, 3226, 355, 264, 1280, 268, 583, 312, 355, 312, 48, 693, 2418, 264, 1026, 268, 2418, 567, 48, 446, 1388, 1103, 3382, 48, 319, 602, 886, 268, 338, 338, 394, 46, 320, 848, 46, 264, 1294, 341, 3336, 3336, 430, 485, 12, 36, 2896, 571, 268, 327, 46, 320, 744, 46, 585, 439, 48, 1709, 312, 312, 3215, 428, 5989, 1718, 403, 693, 543, 2655, 48, 342, 681, 264, 474, 284, 430, 1332, 46, 416, 430, 268, 399, 813, 501, 525, 1571, 1571, 48, 12, 1761, 642, 46, 693, 665, 1880, 389, 389, 693, 747, 338, 7959, 48, 342, 766, 693, 348, 692, 268, 692, 766, 264, 2754 ] }, { "model": "ncm", "seed": 42, "index": 7, "text": "\n, house a Tom, in a white little field. lived a big bird with her away. She loved to play and sing a happy song.\nOne day, Chirpy found a new friend, little girl Tina Tina came to visit visit. \"Let's sing sing!\" and said two friends friends. \"I like! You have happy? And And is my time! Come are so pretty!\" Luna and. Luna smiled, and said, \"Yes, please!\" They read share our songs.\"\nBellaBella and her friends played together together. They played with all all their day and had fun fun together.\n", "ids": [ 12, 46, 612, 260, 423, 46, 316, 260, 1610, 398, 1729, 48, 640, 260, 348, 426, 331, 338, 581, 48, 342, 511, 267, 328, 268, 1122, 260, 384, 1296, 48, 12, 391, 330, 46, 2388, 514, 260, 552, 359, 46, 398, 486, 3872, 3872, 559, 267, 1708, 1708, 48, 320, 904, 392, 1122, 1122, 421, 268, 327, 1071, 419, 419, 48, 320, 75, 525, 35, 623, 508, 384, 65, 726, 726, 439, 631, 404, 35, 2264, 492, 407, 758, 421, 2291, 268, 48, 2291, 571, 46, 268, 327, 46, 320, 744, 46, 1294, 421, 315, 1303, 845, 1150, 2205, 403, 12, 5123, 5123, 268, 338, 419, 482, 466, 466, 48, 315, 482, 331, 440, 440, 477, 330, 268, 367, 451, 451, 466, 48, 12, 0, 1, 1, 1 ] } ], "recomputed_ppl": { "ncm": { "per_seed": { "42": 100.45904223772496, "0": 95.57766278409314, "1": 95.6736814649737, "2": 98.57095531656978, "3": 98.59929535901298 }, "mean": 97.77612743247491, "sd_sampling_seed": 2.107185000735112, "max_scored_tokens": 150, "unscorable": 0 }, "ar": { "per_seed": { "42": 14.885483249080956, "0": 14.580821038376598, "1": 14.031941107377678, "2": 15.25808914998773, "3": 14.623375352318366 }, "mean": 14.675941979428266, "sd_sampling_seed": 0.4498197524543124, "max_scored_tokens": 152, "unscorable": 0 } }, "benchmarks": { "ncm": { "file": "tinystories-e23-aligned-bench-ncm.json", "sha256": "3fc5b9311ac1c4db4fe9d9dc3fa922a4d54cfd2cb2087463c076877bc318460b", "bytes": 14234, "modes": { "eager": [ { "batch": 1, "first_call_ms": 32.016000244766474, "p50_ms": 12.903478927910328, "p95_ms": 13.141774176619947, "mean_ms": 12.896014517173171, "stdev_ms": 0.13155744509946757, "sequences_per_s": 77.4984797190618, "peak_memory_mb": 184.9171905517578, "raw_ms": [ 12.77133310213685, 12.608041986823082, 13.20083299651742, 12.936500366777182, 12.909249868243933, 12.914541643112898, 12.893041130155325, 12.94400030747056, 12.763625010848045, 13.04187485948205, 12.914291117340326, 13.138665817677975, 12.849375139921904, 12.785499915480614, 12.935334350913763, 12.897707987576723, 12.856625020503998, 12.854999862611294, 12.774957809597254, 12.929792050272226 ], "gpu_before": [ 0 ], "gpu_after": [ 84 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/eager-batch1.npy", "first_output_sha256": "7ac37afa073baeec567598cb5eaca59719ca5001d3f61c3ae89de44d886f77fc" }, { "batch": 16, "first_call_ms": 57.86970816552639, "p50_ms": 50.85129151120782, "p95_ms": 96.63484243210405, "mean_ms": 60.65928749740124, "stdev_ms": 17.179919186257838, "sequences_per_s": 314.64294267675655, "peak_memory_mb": 1114.4677648544312, "raw_ms": [ 47.95433301478624, 48.471041955053806, 48.67391660809517, 48.36683301255107, 50.23229168727994, 50.53841695189476, 50.14016618952155, 52.881000097841024, 49.88075001165271, 50.05162488669157, 51.70983308926225, 51.16416607052088, 50.03679217770696, 62.2010831721127, 65.12995902448893, 71.35133305564523, 99.18812522664666, 90.68670775741339, 78.0269168317318, 96.50045912712812 ], "gpu_before": [ 0 ], "gpu_after": [ 60 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/eager-batch16.npy", "first_output_sha256": "170abc6bd16c8b7df8afc2e7f331f1283192bfd9f024cc2dce730f29d5459592" }, { "batch": 64, "first_call_ms": 304.74158376455307, "p50_ms": 332.0459998212755, "p95_ms": 445.85348179098224, "mean_ms": 344.47331458795816, "stdev_ms": 60.68224453798184, "sequences_per_s": 192.74437889463553, "peak_memory_mb": 2256.573137283325, "raw_ms": [ 337.0649996213615, 342.18529192730784, 335.47341683879495, 375.42179226875305, 423.4023750759661, 355.53720919415355, 411.1348339356482, 495.770082809031, 443.22629226371646, 380.73379127308726, 328.618582803756, 316.81554205715656, 326.1780836619437, 292.6119579933584, 290.4636668972671, 300.5101662129164, 297.48970828950405, 288.2222910411656, 273.30504171550274, 275.30116587877274 ], "gpu_before": [ 100 ], "gpu_after": [ 86 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/eager-batch64.npy", "first_output_sha256": "d8882171c0710afac0b1f0dcb260482a1b111c9b53db2c86363ed729237701b2" } ], "cached": [ { "batch": 1, "first_call_ms": 19.725958351045847, "p50_ms": 14.023562660440803, "p95_ms": 14.331939816474915, "mean_ms": 14.026749995537102, "stdev_ms": 0.22198182665385913, "sequences_per_s": 71.30855576528418, "peak_memory_mb": 182.15007781982422, "raw_ms": [ 14.047875069081783, 14.142457861453295, 14.112167060375214, 13.851749710738659, 14.546125195920467, 14.202166348695755, 14.320666901767254, 13.999250251799822, 13.932415749877691, 14.153833035379648, 13.663666788488626, 14.219167176634073, 13.792125042527914, 13.757917098701, 14.240666758269072, 14.06425004824996, 13.729874975979328, 13.897750061005354, 13.980957679450512, 13.879917096346617 ], "gpu_before": [ 0 ], "gpu_after": [ 68 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached-batch1.npy", "first_output_sha256": "7ac37afa073baeec567598cb5eaca59719ca5001d3f61c3ae89de44d886f77fc" }, { "batch": 16, "first_call_ms": 91.52895910665393, "p50_ms": 58.56681289151311, "p95_ms": 324.14202315267175, "mean_ms": 102.76750200428069, "stdev_ms": 97.5727635596417, "sequences_per_s": 273.1922604297724, "peak_memory_mb": 1112.546911239624, "raw_ms": [ 77.223707921803, 63.76929162070155, 60.76012505218387, 61.46016577258706, 63.1521251052618, 59.72220888361335, 57.41141689941287, 54.73954230546951, 54.153542034327984, 52.20033274963498, 51.26904183998704, 50.404957961291075, 50.30233319848776, 50.7323332130909, 50.53904093801975, 49.39145781099796, 213.87770818546414, 267.12150033563375, 323.09562526643276, 344.0235829912126 ], "gpu_before": [ 0 ], "gpu_after": [ 94 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached-batch16.npy", "first_output_sha256": "393acce8a6840d5d7dbba52417c6be621f214765e4e1bfd39fbcb5777e95f657" }, { "batch": 64, "first_call_ms": 1003.4333332441747, "p50_ms": 300.07400014437735, "p95_ms": 367.1160168247298, "mean_ms": 310.3905583033338, "stdev_ms": 37.03983977882527, "sequences_per_s": 213.28072398544057, "peak_memory_mb": 2413.095251083374, "raw_ms": [ 443.1060408242047, 282.1844997815788, 363.11654187738895, 317.2584171406925, 332.5028750114143, 322.2817908972502, 308.8330002501607, 298.93083311617374, 301.21716717258096, 306.79012509062886, 298.57354098930955, 287.90320781990886, 285.53779190406203, 283.32445910200477, 274.82904214411974, 296.4848750270903, 292.3550410196185, 307.9616245813668, 307.52233415842056, 297.09795815870166 ], "gpu_before": [ 100 ], "gpu_after": [ 73 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached-batch64.npy", "first_output_sha256": "78801a3f3a6de684793c4b66345a2a9deb911d35070d0814c238de201a4d8b7f" } ], "cached+compiled": [ { "batch": 1, "first_call_ms": 2082.878999877721, "p50_ms": 12.529792031273246, "p95_ms": 13.42054142151028, "mean_ms": 12.60687296744436, "stdev_ms": 0.43277266608076537, "sequences_per_s": 79.80978435269229, "peak_memory_mb": 183.255784034729, "raw_ms": [ 12.524917256087065, 12.54954095929861, 13.187415897846222, 13.084417209029198, 13.418958056718111, 13.00533302128315, 13.450625352561474, 12.94637518003583, 12.66458397731185, 12.188042048364878, 12.37491611391306, 12.534666806459427, 12.638290878385305, 12.256416957825422, 12.368500232696533, 12.131082825362682, 12.192917056381702, 12.095834128558636, 12.154375202953815, 12.370250187814236 ], "gpu_before": [ 0 ], "gpu_after": [ 77 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached+compiled-batch1.npy", "first_output_sha256": "7ac37afa073baeec567598cb5eaca59719ca5001d3f61c3ae89de44d886f77fc" }, { "batch": 16, "first_call_ms": 66.31191726773977, "p50_ms": 44.180312659591436, "p95_ms": 47.245650482364, "mean_ms": 44.512143707834184, "stdev_ms": 1.4625084142231837, "sequences_per_s": 362.15225825312126, "peak_memory_mb": 1150.3388118743896, "raw_ms": [ 46.46883299574256, 46.87245888635516, 47.24029218778014, 45.31179089099169, 44.094250071793795, 43.74425020068884, 44.26645766943693, 43.552250135689974, 43.252500239759684, 44.274458196014166, 43.63950015977025, 43.45137486234307, 42.99324983730912, 44.26637524738908, 47.34745807945728, 44.88229192793369, 45.44220771640539, 42.898792307823896, 43.069124687463045, 43.17495785653591 ], "gpu_before": [ 0 ], "gpu_after": [ 90 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached+compiled-batch16.npy", "first_output_sha256": "393acce8a6840d5d7dbba52417c6be621f214765e4e1bfd39fbcb5777e95f657" }, { "batch": 64, "first_call_ms": 261.7011247202754, "p50_ms": 263.59554193913937, "p95_ms": 962.3790021520111, "mean_ms": 493.1896978756413, "stdev_ms": 337.89853255706606, "sequences_per_s": 242.7962154791553, "peak_memory_mb": 2551.963047981262, "raw_ms": [ 231.85958294197917, 220.0178331695497, 214.01637513190508, 218.23266707360744, 232.3167910799384, 231.54175002127886, 235.04520766437054, 246.15483405068517, 239.98420825228095, 252.05362495034933, 275.1374589279294, 568.2927500456572, 1354.911916423589, 941.7193750850856, 847.2129167057574, 810.3201249614358, 848.6832077614963, 760.0329997949302, 733.6448333226144, 402.61550014838576 ], "gpu_before": [ 0 ], "gpu_after": [ 100 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ncm/cached+compiled-batch64.npy", "first_output_sha256": "78801a3f3a6de684793c4b66345a2a9deb911d35070d0814c238de201a4d8b7f" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison. Some optimized GPU outputs differ from eager; this timing comparison is not a quality-equivalence claim.", "all_cold_outputs_equal_eager": false, "cold_output_consistency": [ { "mode": "eager", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 16, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "eager", "batch": 64, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached", "batch": 16, "tokens_equal": false, "different_token_fraction": 0.00341796875 }, { "mode": "cached", "batch": 64, "tokens_equal": false, "different_token_fraction": 0.0040283203125 }, { "mode": "cached+compiled", "batch": 1, "tokens_equal": true, "different_token_fraction": 0.0 }, { "mode": "cached+compiled", "batch": 16, "tokens_equal": false, "different_token_fraction": 0.00341796875 }, { "mode": "cached+compiled", "batch": 64, "tokens_equal": false, "different_token_fraction": 0.0040283203125 } ], "source": "frozen optimized MLX sampler", "checkpoint_files": { "codec.safetensors": { "sha256": "e8e96b13fcef2d16ac7f8b71b0ddf481e0b09226d3e847fc18ba939baa0be145", "bytes": 62038263 }, "generator.safetensors": { "sha256": "9f4db72ff15c0eae2698a79971c48f25b2fcdefa590244e87943d0b0d74933c6", "bytes": 47340186 }, "inference.safetensors": { "sha256": "1bff87883dbc240a3e48b5fbaf617218cfefc3764d8eaf09d86be62844fb90f9", "bytes": 8293 }, "bpe.json": { "sha256": "a849b62441d405a9b550d173f605fc906bebace393cef71a2347ce27550d1875", "bytes": 296367 }, "spec.json": { "sha256": "0fed73a78ee71a37747199e5a3b492ca5e64c15e4aae124ce1dcfa5f40078498", "bytes": 57434 } } }, "ar": { "file": "tinystories-e23-aligned-bench-ar.json", "sha256": "119f0f4bb34627961cbe643ef754d48802042847eb1bfd66669eff7d11bf0f6f", "bytes": 5035, "modes": { "ar-kv-cache": [ { "batch": 1, "first_call_ms": 107.94450016692281, "p50_ms": 111.43820802681148, "p95_ms": 186.69898484367877, "mean_ms": 120.91766258236021, "stdev_ms": 38.38919374048886, "sequences_per_s": 8.97358291834166, "peak_memory_mb": 62.569294929504395, "raw_ms": [ 86.97750000283122, 85.07916610687971, 84.13770888000727, 84.68887489289045, 82.66987511888146, 87.75562513619661, 102.3420412093401, 91.51662513613701, 124.8373338021338, 161.33625013753772, 185.73708273470402, 106.86879232525826, 111.4927502349019, 116.93316698074341, 130.52870891988277, 111.38366581872106, 111.70616699382663, 179.6166249550879, 204.9751249141991, 167.77016734704375 ], "gpu_before": [ 0 ], "gpu_after": [ 77 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ar/ar-kv-cache-batch1.npy", "first_output_sha256": "cbd6c3c546120755bf14e1489f2fe33ddc1462a17520e15c48f2ea05cb49316b" }, { "batch": 16, "first_call_ms": 303.57208335772157, "p50_ms": 160.98789614625275, "p95_ms": 311.9503375608474, "mean_ms": 178.59727737959474, "stdev_ms": 53.68378878442685, "sequences_per_s": 99.38635377571785, "peak_memory_mb": 202.51961517333984, "raw_ms": [ 228.78720797598362, 342.1073751524091, 177.88204131647944, 167.14137513190508, 163.06508425623178, 162.15937538072467, 160.01570923253894, 163.1760410964489, 156.97020897641778, 156.31545893847942, 161.96008305996656, 151.93145908415318, 149.49445938691497, 151.96145931258798, 152.47762482613325, 149.3290001526475, 150.40183393284678, 147.25500019267201, 169.15162513032556, 310.36312505602837 ], "gpu_before": [ 25 ], "gpu_after": [ 89 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ar/ar-kv-cache-batch16.npy", "first_output_sha256": "a45396d9e1e7846ee730d57c8f46360939d43545d2c02f7581a6d2c4a1b43444" }, { "batch": 64, "first_call_ms": 466.26170817762613, "p50_ms": 521.0710414685309, "p95_ms": 2140.0825295131654, "mean_ms": 827.4632188724354, "stdev_ms": 594.9827581902606, "sequences_per_s": 122.82394319904873, "peak_memory_mb": 642.0428695678711, "raw_ms": [ 660.2688343264163, 639.3266250379384, 529.3952082283795, 469.9944998137653, 460.83550015464425, 468.0830002762377, 465.70354141294956, 411.144292447716, 441.5939161553979, 408.51937513798475, 371.5285421349108, 389.0771670266986, 512.7468747086823, 749.1612089797854, 2196.8463752418756, 2137.0949586853385, 1829.1767910122871, 1328.112583141774, 1278.7102502770722, 801.9448332488537 ], "gpu_before": [ 49 ], "gpu_after": [ 99 ], "first_output_file": "runs/tinystories-e23-aligned-bench-ar/ar-kv-cache-batch64.npy", "first_output_sha256": "f0cd3865ee8580331dd485cdbba04bf09ffcf193bddc70b484376fab5430b264" } ] }, "qualification": "Sequential busy-desktop timing; optimized NCM modes are separate from the eager NCM/AR comparison.", "all_cold_outputs_equal_eager": null, "cold_output_consistency": null, "source": "frozen training AR sampler", "checkpoint_files": { "ar.safetensors": { "sha256": "6312453e7abb0b88889ca818bd156c849797d999589f4a6c9a23297affdb88dc", "bytes": 55183975 }, "ar.json": { "sha256": "6d620a1b8ebb73b6f5da8455c585fc2eb004dd248e3d27cf108ba99efebf6dc4", "bytes": 10514 } } } }, "verified": { "all_sample_ids_lengths_vocabulary_and_text_hashes": true, "sample_files_rehashed": true, "checkpoint_config_tokenizer_and_report_hashes_verified": true, "scorer_nll_aggregation_verified": true, "ordered_test_windows_rehashed": true }, "original_report_audit": { "passed": true, "samples": 2560, "scored_rows": 2560, "report_sha256": "2b2902e54cbefa2343da6d31e5a41ee0e2bba00d410b8d7c2485d49d2ee6d234", "audit_source_sha256": "3427dacfbd745eb9f860d08f0e689c8d5add261633c968662bce25afd5fc6cbe" } } }, "verification": [ "training: 44 frozen source files rehashed successfully", "optimized: 38 frozen source files rehashed successfully" ], "limitations": [ "Scorer weight hash and raw corpus hashes are retained declared identities; these large files were not rehashed in this audit.", "No new inference, test-suite execution, human-rating study, data preprocessing or training replication was performed.", "Raw sample JSONL/scorer files and training state remain local; this sanitized extract does not itself distribute the full reproducibility bundle." ], "active_work": { "job": "tinystories-e24-seed1-train-generator", "qualification": "Repeats E21 with generator seed 1 while retaining the same seed-0 codec. E22/E23 seed-1 variants are queued, not results. Status may advance after this snapshot." }, "training_position_contract": { "predicted_code_positions": 640, "eos_prefix_positions": 1, "teacher_forced_positions": 641, "ar_positions": 128, "qualification": "Position count is not a total-compute or FLOP match." }, "training_seed_contract": { "configured_seed": 0, "generator_initialization_and_minibatch_seed": 1, "generator_residual_projection_key": 8, "ar_initialization_and_minibatch_seed": 2, "ar_residual_projection_key": 9, "qualification": "Names seed 0 and seed 1 refer to configured experiment seeds; internal streams use the recorded source offsets." } }