{
"evidence_end": "g1300 evaluation; training log g1319; final 19 updates not saved",
"phases": [
{
"step": 0,
"select_exact": 96,
"ranking_exact": 9,
"select_n": 260,
"ranking_n": 320,
"token_median": 1802.5,
"ranking_unparsed": 168
},
{
"step": 175,
"select_exact": 112,
"ranking_exact": 18,
"select_n": 260,
"ranking_n": 320,
"token_median": 2079.5,
"ranking_unparsed": 2
},
{
"step": 350,
"select_exact": 116,
"ranking_exact": 22,
"select_n": 260,
"ranking_n": 320,
"token_median": 2071.5,
"ranking_unparsed": 0
},
{
"step": 775,
"select_exact": 127,
"ranking_exact": 29,
"select_n": 260,
"ranking_n": 320,
"token_median": 1975.0,
"ranking_unparsed": 0
},
{
"step": 1225,
"select_exact": 122,
"ranking_exact": 28,
"select_n": 260,
"ranking_n": 320,
"token_median": 2031.0,
"ranking_unparsed": 0
},
{
"step": 1300,
"select_exact": 128,
"ranking_exact": 31,
"select_n": 260,
"ranking_n": 320,
"token_median": 1991.5,
"ranking_unparsed": 0
}
],
"data_counts": {
"train": 6153,
"heldout": 290,
"source_available": {
"ranking_v3": 1000,
"ranking_v2": 1999,
"arch170": 170,
"dataflip_v2": 1514,
"mcq_replay": 1760
},
"train_by_source": {
"mcq_replay": 1760,
"ranking_v2": 1939,
"dataflip_v2": 1434,
"ranking_v3": 900,
"arch170": 120
},
"heldout_by_source": {
"arch170": 50,
"dataflip_v2": 80,
"ranking_v2": 60,
"ranking_v3": 100
},
"train_by_task": {
"select": 3314,
"ranking": 2839
},
"heldout_by_task": {
"select": 130,
"ranking": 160
},
"dataflip_v2_pairs": {
"total": 757,
"heldout": 40,
"by_family": {
"multivariate_regression": 167,
"synthetic_tabular_classification": 167,
"spiral_classification": 167,
"univariate_regression": 89,
"bigram_lm": 167
},
"by_node_family": {
"360-2|multivariate_regression": 165,
"a100_perm|synthetic_tabular_classification": 82,
"a100_t1_2|univariate_regression": 20,
"a100_t1_3|multivariate_regression": 2,
"a100_t1_3|synthetic_tabular_classification": 85,
"a100_t1_3|univariate_regression": 28,
"a100_t1|spiral_classification": 167,
"a100_t1|univariate_regression": 41,
"ophis-gpu|bigram_lm": 167
}
},
"mcq_replay_by_old_source": {
"train": {
"v12_981": 805,
"dataflip-orig": 250,
"dataflip-strict": 39,
"v15-root": 86,
"dataflip-cf": 250,
"v15-main": 330
},
"raw_from_train_v3": {
"dataflip-orig": 250,
"dataflip-cf": 250,
"v12_981": 805,
"v15-main": 500,
"dataflip-strict": 39,
"v15-root": 86
}
}
},
"leakage_checks": {
"qid_overlap": 0,
"group_overlap": 0,
"instance_key_overlap": 0,
"dataset_id_overlap": 0,
"heldout_in_sft": 0,
"dataflip_pairs_split": 0
},
"paired_statistics": {
"splits": {
"SFT->g1300|select": {
"n_questions": 130,
"acc_initial": 0.3692,
"acc_final": 0.4923,
"delta_pp": 12.31,
"boot_ci_pp": [
5.77,
19.23
],
"improved": 35,
"worsened": 13,
"unchanged": 82
},
"SFT->g1300|ranking": {
"n_questions": 160,
"acc_initial": 0.0281,
"acc_final": 0.0969,
"delta_pp": 6.88,
"boot_ci_pp": [
3.75,
10.31
],
"improved": 21,
"worsened": 2,
"unchanged": 137
},
"SFT->g1300|all": {
"n_questions": 290,
"acc_initial": 0.181,
"acc_final": 0.2741,
"delta_pp": 9.31,
"boot_ci_pp": [
5.86,
12.93
],
"improved": 56,
"worsened": 15,
"unchanged": 219
},
"SFT->g1300|ranking_inv_parse_matched": {
"n_samples": 149,
"mean_initial": 4.067,
"mean_final": 2.993,
"delta": -1.074,
"boot_ci": [
-1.456,
-0.685
]
},
"SFT->g175|select": {
"n_questions": 130,
"acc_initial": 0.3692,
"acc_final": 0.4308,
"delta_pp": 6.15,
"boot_ci_pp": [
1.15,
11.54
],
"improved": 27,
"worsened": 14,
"unchanged": 89
},
"SFT->g175|ranking": {
"n_questions": 160,
"acc_initial": 0.0281,
"acc_final": 0.0563,
"delta_pp": 2.81,
"boot_ci_pp": [
0.0,
5.94
],
"improved": 13,
"worsened": 4,
"unchanged": 143
},
"SFT->g175|all": {
"n_questions": 290,
"acc_initial": 0.181,
"acc_final": 0.2241,
"delta_pp": 4.31,
"boot_ci_pp": [
1.55,
7.24
],
"improved": 40,
"worsened": 18,
"unchanged": 232
},
"SFT->g175|ranking_inv_parse_matched": {
"n_samples": 148,
"mean_initial": 4.074,
"mean_final": 3.689,
"delta": -0.385,
"boot_ci": [
-0.797,
0.014
]
},
"g175->g1300|select": {
"n_questions": 130,
"acc_initial": 0.4308,
"acc_final": 0.4923,
"delta_pp": 6.15,
"boot_ci_pp": [
0.38,
11.92
],
"improved": 31,
"worsened": 15,
"unchanged": 84
},
"g175->g1300|ranking": {
"n_questions": 160,
"acc_initial": 0.0563,
"acc_final": 0.0969,
"delta_pp": 4.06,
"boot_ci_pp": [
0.62,
7.81
],
"improved": 18,
"worsened": 8,
"unchanged": 134
},
"g175->g1300|all": {
"n_questions": 290,
"acc_initial": 0.2241,
"acc_final": 0.2741,
"delta_pp": 5.0,
"boot_ci_pp": [
1.72,
8.28
],
"improved": 49,
"worsened": 23,
"unchanged": 218
},
"g175->g1300|ranking_inv_parse_matched": {
"n_samples": 318,
"mean_initial": 3.84,
"mean_final": 3.05,
"delta": -0.789,
"boot_ci": [
-1.044,
-0.531
]
},
"g350->g1300|select": {
"n_questions": 130,
"acc_initial": 0.4462,
"acc_final": 0.4923,
"delta_pp": 4.62,
"boot_ci_pp": [
-1.92,
11.15
],
"improved": 27,
"worsened": 16,
"unchanged": 87
},
"g350->g1300|ranking": {
"n_questions": 160,
"acc_initial": 0.0688,
"acc_final": 0.0969,
"delta_pp": 2.81,
"boot_ci_pp": [
-1.25,
6.88
],
"improved": 17,
"worsened": 11,
"unchanged": 132
},
"g350->g1300|all": {
"n_questions": 290,
"acc_initial": 0.2379,
"acc_final": 0.2741,
"delta_pp": 3.62,
"boot_ci_pp": [
0.0,
7.24
],
"improved": 44,
"worsened": 27,
"unchanged": 219
},
"g350->g1300|ranking_inv_parse_matched": {
"n_samples": 320,
"mean_initial": 3.294,
"mean_final": 3.053,
"delta": -0.241,
"boot_ci": [
-0.475,
-0.006
]
}
},
"by_source": {
"arch170": {
"n_questions": 50,
"acc_sft": 0.29,
"acc_g1300": 0.54,
"delta_pp": 25.0,
"boot_ci_pp": [
12.0,
39.0
]
},
"dataflip_v2": {
"n_questions": 80,
"acc_sft": 0.4188,
"acc_g1300": 0.4625,
"delta_pp": 4.38,
"boot_ci_pp": [
-1.87,
10.63
]
},
"ranking_v2": {
"n_questions": 60,
"acc_sft": 0.0667,
"acc_g1300": 0.2,
"delta_pp": 13.33,
"boot_ci_pp": [
6.67,
20.83
]
},
"ranking_v3": {
"n_questions": 100,
"acc_sft": 0.005,
"acc_g1300": 0.035,
"delta_pp": 3.0,
"boot_ci_pp": [
0.5,
6.5
]
}
},
"dataflip_pairs": {
"SFT": {
"n_pairs": 40,
"both_have_correct_sample": 7,
"all_four_correct": 0,
"both_frac": 0.175,
"boot_ci_both_frac": [
0.075,
0.3
]
},
"g1300": {
"n_pairs": 40,
"both_have_correct_sample": 6,
"all_four_correct": 1,
"both_frac": 0.15,
"boot_ci_both_frac": [
0.05,
0.275
]
},
"SFT->g1300_both_correct_delta": {
"delta": -0.025,
"boot_ci": [
-0.15,
0.1
]
}
},
"notes": [
"acc per question = mean of its 2 samples; bootstrap resamples questions (B=10000).",
"g1300 is the last saved eval of v3.2c; multiple checkpoints were repeatedly observed, so pointwise maxima are not independent confirmations.",
"ranking_inv_parse_matched conditions on parseability at both phases; SFT ranking parse rate was ~47%, so this subset is biased toward easier/shorter outputs."
],
"exact_counts": {
"SFT": {
"select_exact": 96,
"ranking_exact": 9
},
"g175": {
"select_exact": 112,
"ranking_exact": 18
},
"g350": {
"select_exact": 116,
"ranking_exact": 22
},
"g775": {
"select_exact": 127,
"ranking_exact": 29
},
"g1225": {
"select_exact": 122,
"ranking_exact": 28
},
"g1300": {
"select_exact": 128,
"ranking_exact": 31
}
}
},
"scorer_group_statistics": {
"dataflip_v2": {
"groups": 778,
"accuracy_mean": 0.43733933161953725,
"accuracy_std_across_groups": 0.42148427504303254,
"all_wrong_fraction": 0.2853470437017995,
"all_correct_fraction": 0.19665809768637532,
"zero_reward_variance_fraction": 0.4794344473007712
},
"mcq_replay": {
"groups": 924,
"accuracy_mean": 0.6366680194805194,
"accuracy_std_across_groups": 0.3978142341390409,
"all_wrong_fraction": 0.12554112554112554,
"all_correct_fraction": 0.30844155844155846,
"zero_reward_variance_fraction": 0.43398268398268397
},
"ranking_v3": {
"groups": 474,
"accuracy_mean": 0.052083333333333336,
"accuracy_std_across_groups": 0.08078072312471173,
"all_wrong_fraction": 0.4831223628691983,
"all_correct_fraction": 0,
"zero_reward_variance_fraction": 0
},
"ranking_v2": {
"groups": 1022,
"accuracy_mean": 0.2189334637964775,
"accuracy_std_across_groups": 0.23250942282641066,
"all_wrong_fraction": 0.2573385518590998,
"all_correct_fraction": 0,
"zero_reward_variance_fraction": 0
},
"arch170": {
"groups": 59,
"accuracy_mean": 0.7590042372881356,
"accuracy_std_across_groups": 0.2573116988467515,
"all_wrong_fraction": 0,
"all_correct_fraction": 0.11864406779661017,
"zero_reward_variance_fraction": 0.11864406779661017
}
},
"group_accuracy_histograms": {
"dataflip_v2": [
324,
34,
28,
28,
23,
38,
14,
23,
41,
225
],
"mcq_replay": [
195,
39,
37,
15,
32,
37,
38,
35,
68,
428
],
"ranking_v2": [
471,
124,
104,
87,
73,
79,
40,
25,
13,
6
],
"ranking_v3": [
394,
53,
18,
5,
3,
1,
0,
0,
0,
0
],
"arch170": [
2,
1,
2,
3,
2,
2,
8,
1,
10,
28
]
},
"group_histogram_edges": [
-0.0001,
0.09992,
0.19994,
0.29996,
0.39998,
0.5,
0.60002,
0.70004,
0.80006,
0.90008,
1.0001
],
"samples": [
{
"question_id": "q_3afce5",
"sample": 0,
"answer": "B",
"full_chars": 7469,
"excerpt": "Let me analyze each choice for the two-spirals binary classification task with 1024 training rows and 2048 test rows. The critical factor here is training budget of only 32768 total samples seen (1024 steps × batch_size 32).\n\nThe two-spirals task requires learning a highly non-linear decision boundary - specifically, it needs to learn how the arm position correlates with the spiral turn. With 1024 training points, this is a relatively easy dataset.\n\nLet me analyze each choice:\n\n**Choice A:**\n- 4\n\n……（中略 3137 字符）……\n\nough capacity for the complex boundary. The lack of residual connections doesn't hurt here since the path is relatively short without residual blocks.\n\nHowever, Choice A might be better because:\n- Residual connections help gradient flow\n- Layer norm on first block stabilizes initialization\n- SiLU is smooth\n\nActually, let me reconsider. With only 1024 training points and 1024 steps, the model needs to learn a lot. The two-spirals problem is known to be quite difficult for single-hidden-layer netw\n\n……（中略 2832 字符）……\n\nes seen, which is about 32 epochs. This is a relatively limited number of epochs for a 5-hidden-layer network on this problem.\n\nI think all three models have enough capacity to solve the two-spirals problem, so the differentiator is optimization:\n- B has the most stable optimization environment (layer norm on first layer, no residual blocks)\n- A has residual connections but layer norm on layer 3 is less stabilizing\n- C has no layer norm at all\n\nI'll go with Choice B.\n\n\nB",
"source_step": 1300
},
{
"question_id": "q_e6673e",
"sample": 0,
"answer": "A",
"full_chars": 5633,
"excerpt": "Let me analyze each choice for the two-spirals binary classification task with:\n- 1024 training samples\n- 2048 test samples\n- 3 full turns of spiral (highly non-linear boundary)\n- Training budget: 1024 steps × 32 batch size = 32768 samples seen (about 32 epochs over training data)\n- Same optimizer: RMSprop, lr=0.0003, weight_decay=1e-05\n\nThe key considerations:\n1. Two-spirals with 3 turns is a highly non-linear problem requiring deep learning\n2. Training budget is limited (only 32 epochs over tr\n\n……（中略 2295 字符）……\n\nELU is smooth but no residual support\n- The lack of residuals with 5 hidden layers might prevent optimization from reaching useful capacity\n- However, with GELU (monotonic), it might help with optimization\n\n**Choice C** with residual connections but NO LayerNorm and LeakyReLU:\n- Residual connections help\n- No LayerNorm could mean less regularization and potentially higher risk of overfitting\n- LeakyReLU is smooth but without LayerNorm and residual connections, it might have gradient issues\n- No \n\n……（中略 1838 字符）……\n\no LayerNorm and no residuals... That could lead to problematic gradient flow. The gradient might not flow well through the first layer into the subsequent layers without normalization.\n\nI'll go with Choice A as my answer. It combines the best of several approaches: residual connections for optimization, LayerNorm for stable training, and SiLU for the highly non-linear boundary. The model depth (4 vs 5 hidden layers) adapts appropriately for the constraint of 3 turns.\n\n\nA",
"source_step": 1300
},
{
"question_id": "q_797755",
"sample": 0,
"answer": "A",
"full_chars": 7463,
"excerpt": "Let me analyze the three choices for this bigram language modeling task.\n\nThe task is bigram language modeling with:\n- Vocab size: 32\n- Context length: 12\n- Only needs to learn P(y|x) - a 32×32 transition matrix\n- Training budget: 1024 steps × 32 batch size = 32768 samples seen\n- Train runs: 800 samples\n- Each choice uses Adagrad with lr=0.0001, wd=0.0001\n\nKey considerations:\n\n1. **Task complexity**: This is a simple bigram task - only the previous token matters. The true underlying distribution\n\n……（中略 1970 字符）……\n\nrge fastest:\n\n1. **Depth vs Width**: In Peter Silver's Law of Parallelism, faster convergence can happen with moderate depth. However, with Adagrad's fast decay, deeper networks might converge slower.\n\n2. **Head optimization**: Choice B has a single-head architecture that might have headhole issues. The head for C is 32→32 (identity like), which is good. Head A has 128→32.\n\n3. **Actually, reconsider**: With Adagrad at lr=0.0001, the effective learning rate is very small. The model needs to take \n\n……（中略 3993 字符）……\n\nmall vocab\n\nIn Practice, with Adagrad this low learning rate and limited steps, I think the model that can fit the solution fastest wins.\n\nThe transition matrix has balloons at positions 0-31, and the network needs to converge on whatever is left.\n\nI'll go with Choice A (d_model=128, 3 layers) as the winner. It has the most capacity per layer, a reasonable depth for this type of task, and the wider FFN provides enough dimensionality to represent the bigram structure.\n\n\nA",
"source_step": 1300
}
],
"raw_sha256": {
"raw_step0.jsonl": "cc2728eb9268136e703fe8bda83968296e4b0068019c71e93610064fcf07d467",
"raw_step175.jsonl": "3a838cc040dcd23a5c4b4b6f1f521b3e6e2708d52e34e5fe41644a6ef2a6248f",
"g350.jsonl": "52c7d6d19e9f415155fe8b669ecda27ff8a7652b8c1979fa48f67ff5b2b9612c",
"g775.jsonl": "f589c26d26302b76e9192e1ae5ae33f43d621844a7a5b47086e8fc192820bc6d",
"g1225.jsonl": "1b6dbfce06008996db56325b38f1d7b54f7024b448cf502ba547f370ede78947",
"g1300.jsonl": "f949080ac7b70d60dca0052392dd6920284c4422db4a97cc761351a3b45dcce5"
},
"group_scope": "Completed scorer groups, including possible subsequent stale drops; optimizer batch not reconstructed."
}