{
  "run_id": "nglab1x_input_v2_fixed",
  "injection_position": "input",
  "steps": 2000,
  "seed": 42,
  "epoch_batches": 0,
  "fixed_train_probe_sha256": "38d1254a827759d6",
  "fixed_train_probe_batches": 4,
  "probe_eval_interval": 10,
  "fixed_train_loss_log": "fixed_train_loss.jsonl",
  "fixed_train_freq_bin_log": "fixed_train_freq_bin_loss.jsonl",
  "exact_freq_log": "exact_freq_loss.jsonl",
  "freq_index": "/data/home/guoshaoyang/ngram-gap-lab/data/freq_index.npz",
  "compute_dtype": "bf16",
  "torch_compile": true,
  "final_train_loss": 1.6058354377746582,
  "final_val_loss": 6.8136595487594604,
  "final_gap": 5.207824110984802,
  "final_fixed_train_loss": 1.7975293099880219,
  "final_fixed_val_loss": 6.8136595487594604,
  "final_fixed_gap": 5.016130238771439,
  "n_params": 2883141120,
  "config": {
    "vocab_size": 8192,
    "n_layer": 8,
    "n_head": 6,
    "n_embd": 768,
    "sequence_len": 2048,
    "dropout": 0.0,
    "bias": true,
    "enable_nanogpt_ngram_ve": true,
    "enable_unigram_ve": false,
    "enable_bigram_ve": true,
    "enable_trigram_ve": true,
    "enable_fourgram_ve": false,
    "nanogpt_ngram_injection_position": "input",
    "nanogpt_adam_lr": 0.004,
    "ngram_table_betas": [
      0.0,
      0.99
    ],
    "table_optimizer": "rmsprop",
    "table_lr_scale": 2.0,
    "table_betas": [
      0.0,
      0.99
    ],
    "table_mult": 64,
    "intervention": "none",
    "intervention_epoch": 1,
    "adam_betas": [
      0.8,
      0.95
    ],
    "weight_decay": 0.1,
    "seed": 42,
    "max_steps": 2000,
    "device_batch_size": 72,
    "total_batch_size": 147456,
    "val_interval_steps": 10,
    "val_batches": 4,
    "table_norm_interval_steps": 10,
    "warmdown_ratio": 0.65,
    "lr_schedule_epochs": 0,
    "data_dir": "/data/home/guoshaoyang/ngram-gap-lab/data/tokenized",
    "train_shards": [
      1
    ],
    "val_shards": [
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      6542
    ],
    "data_mode": "fixed",
    "data_seed": 42,
    "epoch_batches": 0,
    "out_dir": "/data/home/guoshaoyang/ngram-gap-lab/data/runs_fixed/nglab1x_input_v2_fixed"
  }
}