ArchitectureIQ 参考文献与引用规划

@article{zoph2017nasrl,
  author    = {Barret Zoph and Quoc V. Le},
  title     = {Neural Architecture Search with Reinforcement Learning},
  journal   = {arXiv preprint arXiv:1611.01578},
  year      = {2016},
  eprint    = {1611.01578},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1611.01578},
}

@article{pham2018enas,
  author    = {Hieu Pham and Melody Y. Guan and Barret Zoph and Quoc V. Le and Jeff Dean},
  title     = {Efficient Neural Architecture Search via Parameter Sharing},
  journal   = {arXiv preprint arXiv:1802.03268},
  year      = {2018},
  eprint    = {1802.03268},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1802.03268},
}

@inproceedings{liu2019darts,
  author    = {Hanxiao Liu and Karen Simonyan and Yiming Yang},
  title     = {DARTS: Differentiable Architecture Search},
  booktitle = {ICLR 2019},
  year      = {2018},
  eprint    = {1806.09055},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1806.09055},
  note      = {Code and pretrained models available at https://github.com/quark0/darts},
}

@inproceedings{cai2020ofa,
  author    = {Han Cai and Chuang Gan and Tianzhe Wang and Zhekai Zhang and Song Han},
  title     = {Once-for-All: Train One Network and Specialize it for Efficient Deployment},
  booktitle = {ICLR 2020},
  year      = {2019},
  eprint    = {1908.09791},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1908.09791},
}

@article{ying2019nasbench101,
  author    = {Chris Ying and Aaron Klein and Esteban Real and Eric Christiansen and Kevin Murphy and Frank Hutter},
  title     = {NAS-Bench-101: Towards Reproducible Neural Architecture Search},
  journal   = {arXiv preprint arXiv:1902.09635},
  year      = {2019},
  eprint    = {1902.09635},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1902.09635},
}

@inproceedings{dong2020nasbench201,
  author    = {Xuanyi Dong and Yi Yang},
  title     = {NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search},
  booktitle = {ICLR 2020},
  year      = {2020},
  eprint    = {2001.00326},
  archivePrefix = {arXiv},
  primaryClass = {cs.CV},
  url       = {https://arxiv.org/abs/2001.00326},
  note      = {As a spotlight paper; 16 pages; 10 figures; 7 tables; Code is available at https://gith…},
}

@article{klein2019tabular,
  author    = {Aaron Klein and Frank Hutter},
  title     = {Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization},
  journal   = {arXiv preprint arXiv:1905.04970},
  year      = {2019},
  eprint    = {1905.04970},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1905.04970},
}

@article{eggensperger2021hpobench,
  author    = {Katharina Eggensperger and Philipp Müller and Neeratyoy Mallik and Matthias Feurer and René Sass and Aaron Klein and Noor Awad and Marius Lindauer and Frank Hutter},
  title     = {HPOBench: A Collection of Reproducible Multi-Fidelity Benchmark Problems for HPO},
  journal   = {arXiv preprint arXiv:2109.06716},
  year      = {2021},
  eprint    = {2109.06716},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2109.06716},
}

@article{pfisterer2022yahpo,
  author    = {Florian Pfisterer and Lennart Schneider and Julia Moosbauer and Martin Binder and Bernd Bischl},
  title     = {YAHPO Gym -- An Efficient Multi-Objective Multi-Fidelity Benchmark for Hyperparameter Optimization},
  journal   = {arXiv preprint arXiv:2109.03670},
  year      = {2021},
  eprint    = {2109.03670},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2109.03670},
}

@inproceedings{white2021predictors,
  author    = {Colin White and Arber Zela and Binxin Ru and Yang Liu and Frank Hutter},
  title     = {How Powerful are Performance Predictors in Neural Architecture Search?},
  booktitle = {NeurIPS 2021},
  year      = {2021},
  eprint    = {2104.01177},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2104.01177},
}

@inproceedings{white2019bananas,
  author    = {Colin White and Willie Neiswanger and Yash Savani},
  title     = {BANANAS: Bayesian Optimization with Neural Architectures for Neural Architecture Search},
  booktitle = {AAAI Conference on Artificial Intelligence 2021},
  year      = {2019},
  eprint    = {1910.11858},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1910.11858},
}

@inproceedings{abdelfattah2021zerocost,
  author    = {Mohamed S. Abdelfattah and Abhinav Mehrotra and Łukasz Dudziak and Nicholas D. Lane},
  title     = {Zero-Cost Proxies for Lightweight NAS},
  booktitle = {ICLR 2021},
  year      = {2021},
  eprint    = {2101.08134},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2101.08134},
}

@inproceedings{mellor2021naswot,
  author    = {Joseph Mellor and Jack Turner and Amos Storkey and Elliot J. Crowley},
  title     = {Neural Architecture Search without Training},
  booktitle = {ICML 2021},
  year      = {2020},
  eprint    = {2006.04647},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2006.04647},
}

@article{unterthiner2020predicting,
  author    = {Thomas Unterthiner and Daniel Keysers and Sylvain Gelly and Olivier Bousquet and Ilya Tolstikhin},
  title     = {Predicting Neural Network Accuracy from Weights},
  journal   = {arXiv preprint arXiv:2002.11448},
  year      = {2020},
  eprint    = {2002.11448},
  archivePrefix = {arXiv},
  primaryClass = {stat.ML},
  url       = {https://arxiv.org/abs/2002.11448},
}

@inproceedings{domhan2015learningcurves,
  author    = {Tobias Domhan and Jost Tobias Springenberg and Frank Hutter},
  title     = {Speeding Up Automatic Hyperparameter Optimization of Deep Neural Networks by Extrapolation of Learning Curves},
  booktitle = {IJCAI, pp. 3462-3468},
  year      = {2015},
  url       = {https://www.ijcai.org/Proceedings/15/Papers/525.pdf},
}

@article{mohr2022learningcurves,
  author    = {Felix Mohr and Jan N. van Rijn},
  title     = {Learning Curves for Decision Making in Supervised Machine Learning: A Survey},
  journal   = {Machine Learning, Volume 113, pages 8371-8425 (2024)},
  year      = {2022},
  eprint    = {2201.12150},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2201.12150},
}

@article{bischl2021hpo,
  author    = {Bernd Bischl and Martin Binder and Michel Lang and Tobias Pielok and Jakob Richter and Stefan Coors and Janek Thomas and Theresa Ullmann and Marc Becker and Anne-Laure Boulesteix and Difan Deng and Marius Lindauer},
  title     = {Hyperparameter Optimization: Foundations, Algorithms, Best Practices and Open Challenges},
  journal   = {arXiv preprint arXiv:2107.05847},
  year      = {2021},
  eprint    = {2107.05847},
  archivePrefix = {arXiv},
  primaryClass = {stat.ML},
  url       = {https://arxiv.org/abs/2107.05847},
}

@article{vanrijn2018hpi,
  author    = {J. N. van Rijn and F. Hutter},
  title     = {Hyperparameter Importance Across Datasets},
  journal   = {arXiv preprint arXiv:1710.04725},
  year      = {2017},
  eprint    = {1710.04725},
  archivePrefix = {arXiv},
  primaryClass = {stat.ML},
  url       = {https://arxiv.org/abs/1710.04725},
}

@article{bengio2012practical,
  author    = {Yoshua Bengio},
  title     = {Practical recommendations for gradient-based training of deep architectures},
  journal   = {arXiv preprint arXiv:1206.5533},
  year      = {2012},
  eprint    = {1206.5533},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1206.5533},
}

@article{chan2024mlebench,
  author    = {Jun Shern Chan and Neil Chowdhury and Oliver Jaffe and James Aung and Dane Sherburn and Evan Mays and Giulio Starace and Kevin Liu and Leon Maksin and Tejal Patwardhan and Lilian Weng and Aleksander Mądry},
  title     = {MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering},
  journal   = {arXiv preprint arXiv:2410.07095},
  year      = {2024},
  eprint    = {2410.07095},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2410.07095},
}

@article{huang2023mlagentbench,
  author    = {Qian Huang and Jian Vora and Percy Liang and Jure Leskovec},
  title     = {MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation},
  journal   = {arXiv preprint arXiv:2310.03302},
  year      = {2023},
  eprint    = {2310.03302},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2310.03302},
}

@article{nathani2025mlgym,
  author    = {Deepak Nathani and Lovish Madaan and Nicholas Roberts and Nikolay Bashlykov and Ajay Menon and Vincent Moens and Amar Budhiraja and Despoina Magka and Vladislav Vorotilov and Gaurav Chaurasia and Dieuwke Hupkes and Ricardo Silveira Cabral and Tatiana Shavrina and Jakob Foerster and Yoram Bachrach and William Yang Wang and Roberta Raileanu},
  title     = {MLGym: A New Framework and Benchmark for Advancing AI Research Agents},
  journal   = {arXiv preprint arXiv:2502.14499},
  year      = {2025},
  eprint    = {2502.14499},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2502.14499},
}

@article{wijk2024rebench,
  author    = {Hjalmar Wijk and Tao Lin and Joel Becker and Sami Jawhar and Neev Parikh and Thomas Broadley and Lawrence Chan and Michael Chen and Josh Clymer and Jai Dhyani and Elena Ericheva and Katharyn Garcia and Brian Goodrich and Nikola Jurkovic and Holden Karnofsky and Megan Kinniment and Aron Lajko and Seraphina Nix and Lucas Sato and William Saunders and Maksym Taran and Ben West and Elizabeth Barnes},
  title     = {RE-Bench: Evaluating frontier AI R\&D capabilities of language model agents against human experts},
  journal   = {arXiv preprint arXiv:2411.15114},
  year      = {2024},
  eprint    = {2411.15114},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2411.15114},
}

@article{paperbench2025,
  author    = {Giulio Starace and Oliver Jaffe and Dane Sherburn and James Aung and Jun Shern Chan and Leon Maksin and Rachel Dias and Evan Mays and Benjamin Kinsella and Wyatt Thompson and Johannes Heidecke and Amelia Glaese and Tejal Patwardhan},
  title     = {PaperBench: Evaluating AI's Ability to Replicate AI Research},
  journal   = {arXiv preprint arXiv:2504.01848},
  year      = {2025},
  eprint    = {2504.01848},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2504.01848},
}

@inproceedings{chen2025mlrbench,
  author    = {Hui Chen and Miao Xiong and Yujie Lu and Wei Han and Ailin Deng and Yufei He and Jiaying Wu and Yibo Li and Yue Liu and Bryan Hooi},
  title     = {MLR-Bench: Evaluating AI Agents on Open-Ended Machine Learning Research},
  booktitle = {NeurIPS 2025},
  year      = {2025},
  eprint    = {2505.19955},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2505.19955},
}

@article{liang2024mlbench,
  author    = {Xiangru Tang and Yuliang Liu and Zefan Cai and Yanjun Shao and Junjie Lu and Yichi Zhang and Zexuan Deng and Helan Hu and Kaikai An and Ruijun Huang and Shuzheng Si and Sheng Chen and Haozhe Zhao and Liang Chen and Yan Wang and Tianyu Liu and Zhiwei Jiang and Baobao Chang and Yin Fang and Yujia Qin and Wangchunshu Zhou and Yilun Zhao and Arman Cohan and Mark Gerstein},
  title     = {ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code},
  journal   = {arXiv preprint arXiv:2311.09835},
  year      = {2023},
  eprint    = {2311.09835},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2311.09835},
}

@article{mlsbench2026,
  author    = {Bohan Lyu and Yucheng Yang and Siqiao Huang and Jiaru Zhang and Qixin Xu and Xinghan Li and Xinyang Han and Yicheng Zhang and Huaqing Zhang and Runhan Huang and Kaicheng Yang and Zitao Chen and Wentao Guo and Junlin Yang and Xinyue Ai and Wenhao Chai and Yadi Cao and Ziran Yang and Kun Wang and Dapeng Jiang and Huan-ang Gao and Shange Tang and Chengshuai Shi and Simon S. Du and Max Simchowitz and Jiantao Jiao and Dawn Song and Chi Jin},
  title     = {MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI},
  journal   = {arXiv preprint arXiv:2605.08678},
  year      = {2026},
  eprint    = {2605.08678},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2605.08678},
}

@article{autolab2026,
  author    = {Zhangchen Xu and Junda Chen and Yue Huang and Dongfu Jiang and Jiefeng Chen and Hang Hua and Zijian Wu and Zheyuan Liu and Zexue He and Lichi Li and Shizhe Diao and Jiaxin Pei and Jinsung Yoon and Hao Zhang and Mengdi Wang and Radha Poovendran and Misha Sra and Alex Pentland and Zichen Chen},
  title     = {AutoLab: Can Frontier Models Solve Long-Horizon Auto Research and Engineering Tasks?},
  journal   = {arXiv preprint arXiv:2606.05080},
  year      = {2026},
  eprint    = {2606.05080},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2606.05080},
}

@inproceedings{jimenez2024swebench,
  author    = {Carlos E. Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik Narasimhan},
  title     = {SWE-bench: Can Language Models Resolve Real-World GitHub Issues?},
  booktitle = {ICLR 2024},
  year      = {2023},
  eprint    = {2310.06770},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2310.06770},
  note      = {Code, and leaderboard are available at https://www.swebench.com , https://openreview.ne…},
}

@inproceedings{metr2025timehorizon,
  author    = {Thomas Kwa and Ben West and Joel Becker and Amy Deng and Katharyn Garcia and Max Hasin and Sami Jawhar and Megan Kinniment and Nate Rush and Sydney Von Arx and Ryan Bloom and Thomas Broadley and Haoxing Du and Brian Goodrich and Nikola Jurkovic and Luke Harold Miles and Seraphina Nix and Tao Lin and Chris Painter and Neev Parikh and David Rein and Lucas Jun Koba Sato and Hjalmar Wijk and Daniel M. Ziegler and Elizabeth Barnes and Lawrence Chan},
  title     = {Measuring AI Ability to Complete Long Software Tasks},
  booktitle = {NeurIPS 2025},
  year      = {2025},
  eprint    = {2503.14499},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2503.14499},
}

@inproceedings{chen2024scienceagentbench,
  author    = {Ziru Chen and Shijie Chen and Yuting Ning and Qianheng Zhang and Boshi Wang and Botao Yu and Yifei Li and Zeyi Liao and Chen Wei and Zitong Lu and Vishal Dey and Mingyi Xue and Frazier N. Baker and Benjamin Burns and Daniel Adu-Ampratwum and Xuhui Huang and Xia Ning and Song Gao and Yu Su and Huan Sun},
  title     = {ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery},
  booktitle = {ICLR 2025},
  year      = {2024},
  eprint    = {2410.05080},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2410.05080},
}

@article{majumder2024discoverybench,
  author    = {Bodhisattwa Prasad Majumder and Harshit Surana and Dhruv Agarwal and Bhavana Dalvi Mishra and Abhijeetsingh Meena and Aryan Prakhar and Tirth Vora and Tushar Khot and Ashish Sabharwal and Peter Clark},
  title     = {DiscoveryBench: Towards Data-Driven Discovery with Large Language Models},
  journal   = {arXiv preprint arXiv:2407.01725},
  year      = {2024},
  eprint    = {2407.01725},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2407.01725},
}

@article{chi2026ai4aibench,
  author    = {Yizhe Chi and Wenyi Li and Deyao Hong and Xiaoqiu Wang and Mingju Gao and Kaisen Yang and Bingxiang He and Youjie Zheng and Calvin Xiao and Qinhuai Na},
  title     = {AI4AI-Bench: Benchmarking LLM Agents in Algorithmic Design for Recursive Self-Improvement},
  journal   = {arXiv preprint arXiv:2608.20318},
  year      = {2026},
  eprint    = {2608.20318},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2608.20318},
}

@article{rsibenchdata2026,
  author    = {Fanqing Meng and Lingxiao Du and Qiguang Chen and Ziqi Zhao and Haocheng Lu and Mengkang Hu and Michael Qizhe Shieh},
  title     = {RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement},
  journal   = {arXiv preprint arXiv:2607.25886},
  year      = {2026},
  eprint    = {2607.25886},
  archivePrefix = {arXiv},
  primaryClass = {cs.SE},
  url       = {https://arxiv.org/abs/2607.25886},
}

@article{posttrainbench2026,
  author    = {Ben Rank and Hardik Bhatnagar and Ameya Prabhu and Shira Eisenberg and Karina Nguyen and Matthias Bethge and Maksym Andriushchenko},
  title     = {PostTrainBench: Can LLM Agents Automate LLM Post-Training?},
  journal   = {arXiv preprint arXiv:2603.08640},
  year      = {2026},
  eprint    = {2603.08640},
  archivePrefix = {arXiv},
  primaryClass = {cs.SE},
  url       = {https://arxiv.org/abs/2603.08640},
}

@article{rsisurvey2026,
  author    = {Mingguang Chen and Licheng Wang and Bo Qu},
  title     = {Recursive Self-Improvement in AI: From Bounded Self-Refinement to Autonomous Research Loops},
  journal   = {arXiv preprint arXiv:2607.07663},
  year      = {2026},
  eprint    = {2607.07663},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2607.07663},
}

@article{tao2025selfevolving,
  author    = {Huan-ang Gao and Jiayi Geng and Wenyue Hua and Mengkang Hu and Xinzhe Juan and Hongzhang Liu and Shilong Liu and Jiahao Qiu and Xuan Qi and Yiran Wu and Hongru Wang and Han Xiao and Yuhang Zhou and Shaokun Zhang and Jiayi Zhang and Jinyu Xiang and Yixiong Fang and Qiwen Zhao and Dongrui Liu and Qihan Ren and Cheng Qian and Zhenhailong Wang and Minda Hu and Huazheng Wang and Qingyun Wu and Heng Ji and Mengdi Wang},
  title     = {A Survey of Self-Evolving Agents: What, When, How, and Where to Evolve on the Path to Artificial Super Intelligence},
  journal   = {arXiv preprint arXiv:2507.21046},
  year      = {2025},
  eprint    = {2507.21046},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2507.21046},
}

@article{zhang2025dgm,
  author    = {Jenny Zhang and Shengran Hu and Cong Lu and Robert Lange and Jeff Clune},
  title     = {Darwin Godel Machine: Open-Ended Evolution of Self-Improving Agents},
  journal   = {arXiv preprint arXiv:2505.22954},
  year      = {2025},
  eprint    = {2505.22954},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2505.22954},
}

@inproceedings{yin2024godelagent,
  author    = {Xunjian Yin and Xinyi Wang and Liangming Pan and Li Lin and Xiaojun Wan and William Yang Wang},
  title     = {Gödel Agent: A Self-Referential Agent Framework for Recursive Self-Improvement},
  booktitle = {ACL 2025},
  year      = {2024},
  eprint    = {2410.04444},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2410.04444},
  note      = {Main. The code can be found at https://github.com/Arvid-pku/Godel_Agent},
}

@inproceedings{zelikman2023stop,
  author    = {Eric Zelikman and Eliana Lorch and Lester Mackey and Adam Tauman Kalai},
  title     = {Self-Taught Optimizer (STOP): Recursively Self-Improving Code Generation},
  booktitle = {COLM 2024},
  year      = {2023},
  eprint    = {2310.02304},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2310.02304},
}

@article{novikov2025alphaevolve,
  author    = {Alexander Novikov and Ngân Vũ and Marvin Eisenberger and Emilien Dupont and Po-Sen Huang and Adam Zsolt Wagner and Sergey Shirobokov and Borislav Kozlovskii and Francisco J. R. Ruiz and Abbas Mehrabian and M. Pawan Kumar and Abigail See and Swarat Chaudhuri and George Holland and Alex Davies and Sebastian Nowozin and Pushmeet Kohli and Matej Balog},
  title     = {AlphaEvolve: A coding agent for scientific and algorithmic discovery},
  journal   = {arXiv preprint arXiv:2506.13131},
  year      = {2025},
  eprint    = {2506.13131},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2506.13131},
}

@article{lu2024aiscientist,
  author    = {Chris Lu and Cong Lu and Robert Tjarko Lange and Jakob Foerster and Jeff Clune and David Ha},
  title     = {The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery},
  journal   = {arXiv preprint arXiv:2408.06292},
  year      = {2024},
  eprint    = {2408.06292},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2408.06292},
}

@article{yamada2025aiscientistv2,
  author    = {Yutaro Yamada and Robert Tjarko Lange and Cong Lu and Shengran Hu and Chris Lu and Jakob Foerster and Jeff Clune and David Ha},
  title     = {The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search},
  journal   = {arXiv preprint arXiv:2504.08066},
  year      = {2025},
  eprint    = {2504.08066},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2504.08066},
}

@article{schmidgall2025agentlab,
  author    = {Samuel Schmidgall and Yusheng Su and Ze Wang and Ximeng Sun and Jialian Wu and Xiaodong Yu and Jiang Liu and Michael Moor and Zicheng Liu and Emad Barsoum},
  title     = {Agent Laboratory: Using LLM Agents as Research Assistants},
  journal   = {arXiv preprint arXiv:2501.04227},
  year      = {2025},
  eprint    = {2501.04227},
  archivePrefix = {arXiv},
  primaryClass = {cs.HC},
  url       = {https://arxiv.org/abs/2501.04227},
}

@article{hu2024adas,
  author    = {Shengran Hu and Cong Lu and Jeff Clune},
  title     = {Automated Design of Agentic Systems},
  journal   = {arXiv preprint arXiv:2408.08435},
  year      = {2024},
  eprint    = {2408.08435},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2408.08435},
}

@article{autoresearch2026,
  author    = {Xiang Liu and Qumeng Sun and  Bruno and Xiao Zhang and Jiahao Li and Haoyang Zhang},
  title     = {AutoResearch: Insight In, Hallucination Out},
  journal   = {arXiv preprint arXiv:2608.17906},
  year      = {2026},
  eprint    = {2608.17906},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2608.17906},
}

@article{mlevolve2026,
  author    = {Shangheng Du and Xiangchao Yan and Jinxin Shi and Zongsheng Cao and Shiyang Feng and Zichen Liang and Boyuan Sun and Tianshuo Peng and Yifan Zhou and Xin Li and Jie Zhou and Liang He and Bo Zhang and Lei Bai},
  title     = {MLEvolve: A Self-Evolving Framework for Automated Machine Learning Algorithm Discovery},
  journal   = {arXiv preprint arXiv:2606.06473},
  year      = {2026},
  eprint    = {2606.06473},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2606.06473},
}

@article{execgrounded2026,
  author    = {Chenglei Si and Zitong Yang and Yejin Choi and Emmanuel Candès and Diyi Yang and Tatsunori Hashimoto},
  title     = {Towards Execution-Grounded Automated AI Research},
  journal   = {arXiv preprint arXiv:2601.14525},
  year      = {2026},
  eprint    = {2601.14525},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2601.14525},
}

@inproceedings{tuneahead2026,
  author    = {Yuxiang Luo and Haonan Long and Chen Wang and Qiqi Duan and Xiaotian Lin and Yanwei Xu and Yuyu Luo and Weikai Yang and Nan Tang},
  title     = {TuneAhead: Predicting Fine-tuning Performance Before Full Training Begins},
  booktitle = {ICML 2026},
  year      = {2026},
  eprint    = {2606.17660},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2606.17660},
  note      = {9 pages, 6 figures, accepted as poster:https://icml.cc/virtual/2026/poster/64847},
}

@inproceedings{predictbefore2026,
  author    = {Jingsheng Zheng and Jintian Zhang and Yujie Luo and Yuren Mao and Yunjun Gao and Lun Du and Huajun Chen and Ningyu Zhang},
  title     = {Can We Predict Before Executing Machine Learning Agents?},
  booktitle = {ACL 2026},
  year      = {2026},
  eprint    = {2601.05930},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2601.05930},
}

@inproceedings{wang2023predictable,
  author    = {Qinyuan Ye and Harvey Yiyun Fu and Xiang Ren and Robin Jia},
  title     = {How Predictable Are Large Language Model Capabilities? A Case Study on BIG-bench},
  booktitle = {EMNLP 2023},
  year      = {2023},
  eprint    = {2305.14947},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2305.14947},
  note      = {Findings. Camera-ready version. Code: https://github.com/INK-USC/predicting-big-bench},
}

@article{liu2024agenthpo,
  author    = {Siyi Liu and Chen Gao and Yong Li},
  title     = {Large Language Model Agent for Hyper-Parameter Optimization},
  journal   = {arXiv preprint arXiv:2402.01881},
  year      = {2024},
  eprint    = {2402.01881},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2402.01881},
}

@inproceedings{liu2024llmbo,
  author    = {Tennison Liu and Nicolás Astorga and Nabeel Seedat and Mihaela van der Schaar},
  title     = {Large Language Models to Enhance Bayesian Optimization},
  booktitle = {ICLR 2024},
  year      = {2024},
  eprint    = {2402.03921},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2402.03921},
}

@inproceedings{yang2023opro,
  author    = {Chengrun Yang and Xuezhi Wang and Yifeng Lu and Hanxiao Liu and Quoc V. Le and Denny Zhou and Xinyun Chen},
  title     = {Large Language Models as Optimizers},
  booktitle = {ICLR 2024},
  year      = {2023},
  eprint    = {2309.03409},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2309.03409},
  note      = {42 pages, 26 figures, 15 tables. Code at https://github.com/google-deepmind/opro},
}

@article{zhang2023automlgpt,
  author    = {Shujian Zhang and Chengyue Gong and Lemeng Wu and Xingchao Liu and Mingyuan Zhou},
  title     = {AutoML-GPT: Automatic Machine Learning with GPT},
  journal   = {arXiv preprint arXiv:2305.02499},
  year      = {2023},
  eprint    = {2305.02499},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2305.02499},
}

@inproceedings{hendrycks2021mmlu,
  author    = {Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
  title     = {Measuring Massive Multitask Language Understanding},
  booktitle = {ICLR 2021},
  year      = {2020},
  eprint    = {2009.03300},
  archivePrefix = {arXiv},
  primaryClass = {cs.CY},
  url       = {https://arxiv.org/abs/2009.03300},
  note      = {The test and code is available at https://github.com/hendrycks/test},
}

@article{srivastava2022bigbench,
  author    = {Aarohi Srivastava and Abhinav Rastogi and Abhishek Rao and Abu Awal Md Shoeb and Abubakar Abid and Adam Fisch and Adam R. Brown and Adam Santoro and Aditya Gupta and Adrià Garriga-Alonso and Agnieszka Kluska and Aitor Lewkowycz and Akshat Agarwal and Alethea Power and Alex Ray and Alex Warstadt and Alexander W. Kocurek and Ali Safaya and Ali Tazarv and Alice Xiang and Alicia Parrish and Allen Nie and Aman Hussain and Amanda Askell and Amanda Dsouza and Ambrose Slone and Ameet Rahane and Anantharaman S. Iyer and Anders Andreassen and Andrea Madotto and Andrea Santilli and Andreas Stuhlmüller and Andrew Dai and Andrew La and Andrew Lampinen and Andy Zou and Angela Jiang and Angelica Chen and Anh Vuong and Animesh Gupta and Anna Gottardi and Antonio Norelli and Anu Venkatesh and Arash Gholamidavoodi and Arfa Tabassum and Arul Menezes and Arun Kirubarajan and Asher Mullokandov and Ashish Sabharwal and Austin Herrick and Avia Efrat and Aykut Erdem and Ayla Karakaş and B. Ryan Roberts and Bao Sheng Loe and Barret Zoph and Bartłomiej Bojanowski and Batuhan Özyurt and Behnam Hedayatnia and Behnam Neyshabur and Benjamin Inden and Benno Stein and Berk Ekmekci and Bill Yuchen Lin and Blake Howald and Bryan Orinion and Cameron Diao and Cameron Dour and Catherine Stinson and Cedrick Argueta and César Ferri Ramírez and Chandan Singh and Charles Rathkopf and Chenlin Meng and Chitta Baral and Chiyu Wu and Chris Callison-Burch and Chris Waites and Christian Voigt and Christopher D. Manning and Christopher Potts and Cindy Ramirez and Clara E. Rivera and Clemencia Siro and Colin Raffel and Courtney Ashcraft and Cristina Garbacea and Damien Sileo and Dan Garrette and Dan Hendrycks and Dan Kilman and Dan Roth and Daniel Freeman and Daniel Khashabi and Daniel Levy and Daniel Moseguí González and Danielle Perszyk and Danny Hernandez and Danqi Chen and Daphne Ippolito and Dar Gilboa and David Dohan and David Drakard and David Jurgens and Debajyoti Datta and Deep Ganguli and Denis Emelin and Denis Kleyko and Deniz Yuret and Derek Chen and Derek Tam and Dieuwke Hupkes and Diganta Misra and Dilyar Buzan and Dimitri Coelho Mollo and Diyi Yang and Dong-Ho Lee and Dylan Schrader and Ekaterina Shutova and Ekin Dogus Cubuk and Elad Segal and Eleanor Hagerman and Elizabeth Barnes and Elizabeth Donoway and Ellie Pavlick and Emanuele Rodola and Emma Lam and Eric Chu and Eric Tang and Erkut Erdem and Ernie Chang and Ethan A. Chi and Ethan Dyer and Ethan Jerzak and Ethan Kim and Eunice Engefu Manyasi and Evgenii Zheltonozhskii and Fanyue Xia and Fatemeh Siar and Fernando Martínez-Plumed and Francesca Happé and Francois Chollet and Frieda Rong and Gaurav Mishra and Genta Indra Winata and Gerard de Melo and Germán Kruszewski and Giambattista Parascandolo and Giorgio Mariani and Gloria Wang and Gonzalo Jaimovitch-López and Gregor Betz and Guy Gur-Ari and Hana Galijasevic and Hannah Kim and Hannah Rashkin and Hannaneh Hajishirzi and Harsh Mehta and Hayden Bogar and Henry Shevlin and Hinrich Schütze and Hiromu Yakura and Hongming Zhang and Hugh Mee Wong and Ian Ng and Isaac Noble and Jaap Jumelet and Jack Geissinger and Jackson Kernion and Jacob Hilton and Jaehoon Lee and Jaime Fernández Fisac and James B. Simon and James Koppel and James Zheng and James Zou and Jan Kocoń and Jana Thompson and Janelle Wingfield and Jared Kaplan and Jarema Radom and Jascha Sohl-Dickstein and Jason Phang and Jason Wei and Jason Yosinski and Jekaterina Novikova and Jelle Bosscher and Jennifer Marsh and Jeremy Kim and Jeroen Taal and Jesse Engel and Jesujoba Alabi and Jiacheng Xu and Jiaming Song and Jillian Tang and Joan Waweru and John Burden and John Miller and John U. Balis and Jonathan Batchelder and Jonathan Berant and Jörg Frohberg and Jos Rozen and Jose Hernandez-Orallo and Joseph Boudeman and Joseph Guerr and Joseph Jones and Joshua B. Tenenbaum and Joshua S. Rule and Joyce Chua and Kamil Kanclerz and Karen Livescu and Karl Krauth and Karthik Gopalakrishnan and Katerina Ignatyeva and Katja Markert and Kaustubh D. Dhole and Kevin Gimpel and Kevin Omondi and Kory Mathewson and Kristen Chiafullo and Ksenia Shkaruta and Kumar Shridhar and Kyle McDonell and Kyle Richardson and Laria Reynolds and Leo Gao and Li Zhang and Liam Dugan and Lianhui Qin and Lidia Contreras-Ochando and Louis-Philippe Morency and Luca Moschella and Lucas Lam and Lucy Noble and Ludwig Schmidt and Luheng He and Luis Oliveros Colón and Luke Metz and Lütfi Kerem Şenel and Maarten Bosma and Maarten Sap and Maartje ter Hoeve and Maheen Farooqi and Manaal Faruqui and Mantas Mazeika and Marco Baturan and Marco Marelli and Marco Maru and Maria Jose Ramírez Quintana and Marie Tolkiehn and Mario Giulianelli and Martha Lewis and Martin Potthast and Matthew L. Leavitt and Matthias Hagen and Mátyás Schubert and Medina Orduna Baitemirova and Melody Arnaud and Melvin McElrath and Michael A. Yee and Michael Cohen and Michael Gu and Michael Ivanitskiy and Michael Starritt and Michael Strube and Michał Swędrowski and Michele Bevilacqua and Michihiro Yasunaga and Mihir Kale and Mike Cain and Mimee Xu and Mirac Suzgun and Mitch Walker and Mo Tiwari and Mohit Bansal and Moin Aminnaseri and Mor Geva and Mozhdeh Gheini and Mukund Varma T and Nanyun Peng and Nathan A. Chi and Nayeon Lee and Neta Gur-Ari Krakover and Nicholas Cameron and Nicholas Roberts and Nick Doiron and Nicole Martinez and Nikita Nangia and Niklas Deckers and Niklas Muennighoff and Nitish Shirish Keskar and Niveditha S. Iyer and Noah Constant and Noah Fiedel and Nuan Wen and Oliver Zhang and Omar Agha and Omar Elbaghdadi and Omer Levy and Owain Evans and Pablo Antonio Moreno Casares and Parth Doshi and Pascale Fung and Paul Pu Liang and Paul Vicol and Pegah Alipoormolabashi and Peiyuan Liao and Percy Liang and Peter Chang and Peter Eckersley and Phu Mon Htut and Pinyu Hwang and Piotr Miłkowski and Piyush Patil and Pouya Pezeshkpour and Priti Oli and Qiaozhu Mei and Qing Lyu and Qinlang Chen and Rabin Banjade and Rachel Etta Rudolph and Raefer Gabriel and Rahel Habacker and Ramon Risco and Raphaël Millière and Rhythm Garg and Richard Barnes and Rif A. Saurous and Riku Arakawa and Robbe Raymaekers and Robert Frank and Rohan Sikand and Roman Novak and Roman Sitelew and Ronan LeBras and Rosanne Liu and Rowan Jacobs and Rui Zhang and Ruslan Salakhutdinov and Ryan Chi and Ryan Lee and Ryan Stovall and Ryan Teehan and Rylan Yang and Sahib Singh and Saif M. Mohammad and Sajant Anand and Sam Dillavou and Sam Shleifer and Sam Wiseman and Samuel Gruetter and Samuel R. Bowman and Samuel S. Schoenholz and Sanghyun Han and Sanjeev Kwatra and Sarah A. Rous and Sarik Ghazarian and Sayan Ghosh and Sean Casey and Sebastian Bischoff and Sebastian Gehrmann and Sebastian Schuster and Sepideh Sadeghi and Shadi Hamdan and Sharon Zhou and Shashank Srivastava and Sherry Shi and Shikhar Singh and Shima Asaadi and Shixiang Shane Gu and Shubh Pachchigar and Shubham Toshniwal and Shyam Upadhyay and  Shyamolima and  Debnath and Siamak Shakeri and Simon Thormeyer and Simone Melzi and Siva Reddy and Sneha Priscilla Makini and Soo-Hwan Lee and Spencer Torene and Sriharsha Hatwar and Stanislas Dehaene and Stefan Divic and Stefano Ermon and Stella Biderman and Stephanie Lin and Stephen Prasad and Steven T. Piantadosi and Stuart M. Shieber and Summer Misherghi and Svetlana Kiritchenko and Swaroop Mishra and Tal Linzen and Tal Schuster and Tao Li and Tao Yu and Tariq Ali and Tatsu Hashimoto and Te-Lin Wu and Théo Desbordes and Theodore Rothschild and Thomas Phan and Tianle Wang and Tiberius Nkinyili and Timo Schick and Timofei Kornev and Titus Tunduny and Tobias Gerstenberg and Trenton Chang and Trishala Neeraj and Tushar Khot and Tyler Shultz and Uri Shaham and Vedant Misra and Vera Demberg and Victoria Nyamai and Vikas Raunak and Vinay Ramasesh and Vinay Uday Prabhu and Vishakh Padmakumar and Vivek Srikumar and William Fedus and William Saunders and William Zhang and Wout Vossen and Xiang Ren and Xiaoyu Tong and Xinran Zhao and Xinyi Wu and Xudong Shen and Yadollah Yaghoobzadeh and Yair Lakretz and Yangqiu Song and Yasaman Bahri and Yejin Choi and Yichi Yang and Yiding Hao and Yifu Chen and Yonatan Belinkov and Yu Hou and Yufang Hou and Yuntao Bai and Zachary Seid and Zhuoye Zhao and Zijian Wang and Zijie J. Wang and Zirui Wang and Ziyi Wu},
  title     = {Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models},
  journal   = {Transactions on Machine Learning Research, May/2022, https://openreview.net/forum?id=uyTL5Bvosj},
  year      = {2022},
  eprint    = {2206.04615},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2206.04615},
  note      = {27 pages, 17 figures + references and appendices, repo: https://github.com/google/BIG-b…},
}

@article{liang2022helm,
  author    = {Percy Liang and Rishi Bommasani and Tony Lee and Dimitris Tsipras and Dilara Soylu and Michihiro Yasunaga and Yian Zhang and Deepak Narayanan and Yuhuai Wu and Ananya Kumar and Benjamin Newman and Binhang Yuan and Bobby Yan and Ce Zhang and Christian Cosgrove and Christopher D. Manning and Christopher Ré and Diana Acosta-Navas and Drew A. Hudson and Eric Zelikman and Esin Durmus and Faisal Ladhak and Frieda Rong and Hongyu Ren and Huaxiu Yao and Jue Wang and Keshav Santhanam and Laurel Orr and Lucia Zheng and Mert Yuksekgonul and Mirac Suzgun and Nathan Kim and Neel Guha and Niladri Chatterji and Omar Khattab and Peter Henderson and Qian Huang and Ryan Chi and Sang Michael Xie and Shibani Santurkar and Surya Ganguli and Tatsunori Hashimoto and Thomas Icard and Tianyi Zhang and Vishrav Chaudhary and William Wang and Xuechen Li and Yifan Mai and Yuhui Zhang and Yuta Koreeda},
  title     = {Holistic Evaluation of Language Models},
  journal   = {Published in Transactions on Machine Learning Research (TMLR), 2023},
  year      = {2022},
  eprint    = {2211.09110},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2211.09110},
  note      = {Authored by the Center for Research on Foundation Models (CRFM) at the Stanford Institu…},
}

@article{rein2023gpqa,
  author    = {David Rein and Betty Li Hou and Asa Cooper Stickland and Jackson Petty and Richard Yuanzhe Pang and Julien Dirani and Julian Michael and Samuel R. Bowman},
  title     = {GPQA: A Graduate-Level Google-Proof Q\&A Benchmark},
  journal   = {arXiv preprint arXiv:2311.12022},
  year      = {2023},
  eprint    = {2311.12022},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2311.12022},
}

@article{phan2025hle,
  author    = {Long Phan and Alice Gatti and Ziwen Han and Nathaniel Li and Josephina Hu and Hugh Zhang and Chen Bo Calvin Zhang and Mohamed Shaaban and John Ling and Sean Shi and Michael Choi and Anish Agrawal and Arnav Chopra and Adam Khoja and Ryan Kim and Richard Ren and Jason Hausenloy and Oliver Zhang and Mantas Mazeika and Dmitry Dodonov and Tung Nguyen and Jaeho Lee and Daron Anderson and Mikhail Doroshenko and Alun Cennyth Stokes and Mobeen Mahmood and Oleksandr Pokutnyi and Oleg Iskra and Jessica P. Wang and John-Clark Levin and Mstyslav Kazakov and Fiona Feng and Steven Y. Feng and Haoran Zhao and Michael Yu and Varun Gangal and Chelsea Zou and Zihan Wang and Serguei Popov and Robert Gerbicz and Geoff Galgon and Johannes Schmitt and Will Yeadon and Yongki Lee and Scott Sauers and Alvaro Sanchez and Fabian Giska and Marc Roth and Søren Riis and Saiteja Utpala and Noah Burns and Gashaw M. Goshu and Mohinder Maheshbhai Naiya and Chidozie Agu and Zachary Giboney and Antrell Cheatom and Francesco Fournier-Facio and Sarah-Jane Crowson and Lennart Finke and Zerui Cheng and Jennifer Zampese and Ryan G. Hoerr and Mark Nandor and Hyunwoo Park and Tim Gehrunger and Jiaqi Cai and Ben McCarty and Alexis C Garretson and Edwin Taylor and Damien Sileo and Qiuyu Ren and Usman Qazi and Lianghui Li and Jungbae Nam and John B. Wydallis and Pavel Arkhipov and Jack Wei Lun Shi and Aras Bacho and Chris G. Willcocks and Hangrui Cao and Sumeet Motwani and Emily de Oliveira Santos and Johannes Veith and Edward Vendrow and Doru Cojoc and Kengo Zenitani and Joshua Robinson and Longke Tang and Yuqi Li and Joshua Vendrow and Natanael Wildner Fraga and Vladyslav Kuchkin and Andrey Pupasov Maksimov and Pierre Marion and Denis Efremov and Jayson Lynch and Kaiqu Liang and Aleksandar Mikov and Andrew Gritsevskiy and Julien Guillod and Gözdenur Demir and Dakotah Martinez and Ben Pageler and Kevin Zhou and Saeed Soori and Ori Press and Henry Tang and Paolo Rissone and Sean R. Green and Lina Brüssel and Moon Twayana and Aymeric Dieuleveut and Joseph Marvin Imperial and Ameya Prabhu and Jinzhou Yang and Nick Crispino and Arun Rao and Dimitri Zvonkine and Gabriel Loiseau and Mikhail Kalinin and Marco Lukas and Ciprian Manolescu and Nate Stambaugh and Subrata Mishra and Tad Hogg and Carlo Bosio and Brian P Coppola and Julian Salazar and Jaehyeok Jin and Rafael Sayous and Stefan Ivanov and Philippe Schwaller and Shaipranesh Senthilkuma and Andres M Bran and Andres Algaba and Kelsey Van den Houte and Lynn Van Der Sypt and Brecht Verbeken and David Noever and Alexei Kopylov and Benjamin Myklebust and Bikun Li and Lisa Schut and Evgenii Zheltonozhskii and Qiaochu Yuan and Derek Lim and Richard Stanley and Tong Yang and John Maar and Julian Wykowski and Martí Oller and Anmol Sahu and Cesare Giulio Ardito and Yuzheng Hu and Ariel Ghislain Kemogne Kamdoum and Alvin Jin and Tobias Garcia Vilchis and Yuexuan Zu and Martin Lackner and James Koppel and Gongbo Sun and Daniil S. Antonenko and Steffi Chern and Bingchen Zhao and Pierrot Arsene and Joseph M Cavanagh and Daofeng Li and Jiawei Shen and Donato Crisostomi and Wenjin Zhang and Ali Dehghan and Sergey Ivanov and David Perrella and Nurdin Kaparov and Allen Zang and Ilia Sucholutsky and Arina Kharlamova and Daniil Orel and Vladislav Poritski and Shalev Ben-David and Zachary Berger and Parker Whitfill and Michael Foster and Daniel Munro and Linh Ho and Shankar Sivarajan and Dan Bar Hava and Aleksey Kuchkin and David Holmes and Alexandra Rodriguez-Romero and Frank Sommerhage and Anji Zhang and Richard Moat and Keith Schneider and Zakayo Kazibwe and Don Clarke and Dae Hyun Kim and Felipe Meneguitti Dias and Sara Fish and Veit Elser and Tobias Kreiman and Victor Efren Guadarrama Vilchis and Immo Klose and Ujjwala Anantheswaran and Adam Zweiger and Kaivalya Rawal and Jeffery Li and Jeremy Nguyen and Nicolas Daans and Haline Heidinger and Maksim Radionov and Václav Rozhoň and Vincent Ginis and Christian Stump and Niv Cohen and Rafał Poświata and Josef Tkadlec and Alan Goldfarb and Chenguang Wang and Piotr Padlewski and Stanislaw Barzowski and Kyle Montgomery and Ryan Stendall and Jamie Tucker-Foltz and Jack Stade and T. Ryan Rogers and Tom Goertzen and Declan Grabb and Abhishek Shukla and Alan Givré and John Arnold Ambay and Archan Sen and Muhammad Fayez Aziz and Mark H Inlow and Hao He and Ling Zhang and Younesse Kaddar and Ivar Ängquist and Yanxu Chen and Harrison K Wang and Kalyan Ramakrishnan and Elliott Thornley and Antonio Terpin and Hailey Schoelkopf and Eric Zheng and Avishy Carmi and Ethan D. L. Brown and Kelin Zhu and Max Bartolo and Richard Wheeler and Martin Stehberger and Peter Bradshaw and JP Heimonen and Kaustubh Sridhar and Ido Akov and Jennifer Sandlin and Yury Makarychev and Joanna Tam and Hieu Hoang and David M. Cunningham and Vladimir Goryachev and Demosthenes Patramanis and Michael Krause and Andrew Redenti and David Aldous and Jesyin Lai and Shannon Coleman and Jiangnan Xu and Sangwon Lee and Ilias Magoulas and Sandy Zhao and Ning Tang and Michael K. Cohen and Orr Paradise and Jan Hendrik Kirchner and Maksym Ovchynnikov and Jason O. Matos and Adithya Shenoy and Michael Wang and Yuzhou Nie and Anna Sztyber-Betley and Paolo Faraboschi and Robin Riblet and Jonathan Crozier and Shiv Halasyamani and Shreyas Verma and Prashant Joshi and Eli Meril and Ziqiao Ma and Jérémy Andréoletti and Raghav Singhal and Jacob Platnick and Volodymyr Nevirkovets and Luke Basler and Alexander Ivanov and Seri Khoury and Nils Gustafsson and Marco Piccardo and Hamid Mostaghimi and Qijia Chen and Virendra Singh and Tran Quoc Khánh and Paul Rosu and Hannah Szlyk and Zachary Brown and Himanshu Narayan and Aline Menezes and Jonathan Roberts and William Alley and Kunyang Sun and Arkil Patel and Max Lamparth and Anka Reuel and Linwei Xin and Hanmeng Xu and Jacob Loader and Freddie Martin and Zixuan Wang and Andrea Achilleos and Thomas Preu and Tomek Korbak and Ida Bosio and Fereshteh Kazemi and Ziye Chen and Biró Bálint and Eve J. Y. Lo and Jiaqi Wang and Maria Inês S. Nunes and Jeremiah Milbauer and M Saiful Bari and Zihao Wang and Behzad Ansarinejad and Yewen Sun and Stephane Durand and Hossam Elgnainy and Guillaume Douville and Daniel Tordera and George Balabanian and Hew Wolff and Lynna Kvistad and Hsiaoyun Milliron and Ahmad Sakor and Murat Eron and Andrew Favre D. O. and Shailesh Shah and Xiaoxiang Zhou and Firuz Kamalov and Sherwin Abdoli and Tim Santens and Shaul Barkan and Allison Tee and Robin Zhang and Alessandro Tomasiello and G. Bruno De Luca and Shi-Zhuo Looi and Vinh-Kha Le and Noam Kolt and Jiayi Pan and Emma Rodman and Jacob Drori and Carl J Fossum and Niklas Muennighoff and Milind Jagota and Ronak Pradeep and Honglu Fan and Jonathan Eicher and Michael Chen and Kushal Thaman and William Merrill and Moritz Firsching and Carter Harris and Stefan Ciobâcă and Jason Gross and Rohan Pandey and Ilya Gusev and Adam Jones and Shashank Agnihotri and Pavel Zhelnov and Mohammadreza Mofayezi and Alexander Piperski and David K. Zhang and Kostiantyn Dobarskyi and Roman Leventov and Ignat Soroko and Joshua Duersch and Vage Taamazyan and Andrew Ho and Wenjie Ma and William Held and Ruicheng Xian and Armel Randy Zebaze and Mohanad Mohamed and Julian Noah Leser and Michelle X Yuan and Laila Yacar and Johannes Lengler and Katarzyna Olszewska and Claudio Di Fratta and Edson Oliveira and Joseph W. Jackson and Andy Zou and Muthu Chidambaram and Timothy Manik and Hector Haffenden and Dashiell Stander and Ali Dasouqi and Alexander Shen and Bita Golshani and David Stap and Egor Kretov and Mikalai Uzhou and Alina Borisovna Zhidkovskaya and Nick Winter and Miguel Orbegozo Rodriguez and Robert Lauff and Dustin Wehr and Colin Tang and Zaki Hossain and Shaun Phillips and Fortuna Samuele and Fredrik Ekström and Angela Hammon and Oam Patel and Faraz Farhidi and George Medley and Forough Mohammadzadeh and Madellene Peñaflor and Haile Kassahun and Alena Friedrich and Rayner Hernandez Perez and Daniel Pyda and Taom Sakal and Omkar Dhamane and Ali Khajegili Mirabadi and Eric Hallman and Kenchi Okutsu and Mike Battaglia and Mohammad Maghsoudimehrabani and Alon Amit and Dave Hulbert and Roberto Pereira and Simon Weber and  Handoko and Anton Peristyy and Stephen Malina and Mustafa Mehkary and Rami Aly and Frank Reidegeld and Anna-Katharina Dick and Cary Friday and Mukhwinder Singh and Hassan Shapourian and Wanyoung Kim and Mariana Costa and Hubeyb Gurdogan and Harsh Kumar and Chiara Ceconello and Chao Zhuang and Haon Park and Micah Carroll and Andrew R. Tawfeek and Stefan Steinerberger and Daattavya Aggarwal and Michael Kirchhof and Linjie Dai and Evan Kim and Johan Ferret and Jainam Shah and Yuzhou Wang and Minghao Yan and Krzysztof Burdzy and Lixin Zhang and Antonio Franca and Diana T. Pham and Kang Yong Loh and Joshua Robinson and Abram Jackson and Paolo Giordano and Philipp Petersen and Adrian Cosma and Jesus Colino and Colin White and Jacob Votava and Vladimir Vinnikov and Ethan Delaney and Petr Spelda and Vit Stritecky and Syed M. Shahid and Jean-Christophe Mourrat and Lavr Vetoshkin and Koen Sponselee and Renas Bacho and Zheng-Xin Yong and Florencia de la Rosa and Nathan Cho and Xiuyu Li and Guillaume Malod and Orion Weller and Guglielmo Albani and Leon Lang and Julien Laurendeau and Dmitry Kazakov and Fatimah Adesanya and Julien Portier and Lawrence Hollom and Victor Souza and Yuchen Anna Zhou and Julien Degorre and Yiğit Yalın and Gbenga Daniel Obikoya and  Rai and Filippo Bigi and M. C. Boscá and Oleg Shumar and Kaniuar Bacho and Gabriel Recchia and Mara Popescu and Nikita Shulga and Ngefor Mildred Tanwie and Thomas C. H. Lux and Ben Rank and Colin Ni and Matthew Brooks and Alesia Yakimchyk and  Huanxu and  Liu and Stefano Cavalleri and Olle Häggström and Emil Verkama and Joshua Newbould and Hans Gundlach and Leonor Brito-Santana and Brian Amaro and Vivek Vajipey and Rynaa Grover and Ting Wang and Yosi Kratish and Wen-Ding Li and Sivakanth Gopi and Andrea Caciolai and Christian Schroeder de Witt and Pablo Hernández-Cámara and Emanuele Rodolà and Jules Robins and Dominic Williamson and Vincent Cheng and Brad Raynor and Hao Qi and Ben Segev and Jingxuan Fan and Sarah Martinson and Erik Y. Wang and Kaylie Hausknecht and Michael P. Brenner and Mao Mao and Christoph Demian and Peyman Kassani and Xinyu Zhang and David Avagian and Eshawn Jessica Scipio and Alon Ragoler and Justin Tan and Blake Sims and Rebeka Plecnik and Aaron Kirtland and Omer Faruk Bodur and D. P. Shinde and Yan Carlos Leyva Labrador and Zahra Adoul and Mohamed Zekry and Ali Karakoc and Tania C. B. Santos and Samir Shamseldeen and Loukmane Karim and Anna Liakhovitskaia and Nate Resman and Nicholas Farina and Juan Carlos Gonzalez and Gabe Maayan and Earth Anderson and Rodrigo De Oliveira Pena and Elizabeth Kelley and Hodjat Mariji and Rasoul Pouriamanesh and Wentao Wu and Ross Finocchio and Ismail Alarab and Joshua Cole and Danyelle Ferreira and Bryan Johnson and Mohammad Safdari and Liangti Dai and Siriphan Arthornthurasuk and Isaac C. McAlister and Alejandro José Moyano and Alexey Pronin and Jing Fan and Angel Ramirez-Trinidad and Yana Malysheva and Daphiny Pottmaier and Omid Taheri and Stanley Stepanic and Samuel Perry and Luke Askew and Raúl Adrián Huerta Rodríguez and Ali M. R. Minissi and Ricardo Lorena and Krishnamurthy Iyer and Arshad Anil Fasiludeen and Ronald Clark and Josh Ducey and Matheus Piza and Maja Somrak and Eric Vergo and Juehang Qin and Benjámin Borbás and Eric Chu and Jack Lindsey and Antoine Jallon and I. M. J. McInnis and Evan Chen and Avi Semler and Luk Gloor and Tej Shah and Marc Carauleanu and Pascal Lauer and Tran Đuc Huy and Hossein Shahrtash and Emilien Duc and Lukas Lewark and Assaf Brown and Samuel Albanie and Brian Weber and Warren S. Vaz and Pierre Clavier and Yiyang Fan and Gabriel Poesia Reis e Silva and  Long and  Lian and Marcus Abramovitch and Xi Jiang and Sandra Mendoza and Murat Islam and Juan Gonzalez and Vasilios Mavroudis and Justin Xu and Pawan Kumar and Laxman Prasad Goswami and Daniel Bugas and Nasser Heydari and Ferenc Jeanplong and Thorben Jansen and Antonella Pinto and Archimedes Apronti and Abdallah Galal and Ng Ze-An and Ankit Singh and Tong Jiang and Joan of Arc Xavier and Kanu Priya Agarwal and Mohammed Berkani and Gang Zhang and Zhehang Du and Benedito Alves de Oliveira Junior and Dmitry Malishev and Nicolas Remy and Taylor D. Hartman and Tim Tarver and Stephen Mensah and Gautier Abou Loume and Wiktor Morak and Farzad Habibi and Sarah Hoback and Will Cai and Javier Gimenez and Roselynn Grace Montecillo and Jakub Łucki and Russell Campbell and Asankhaya Sharma and Khalida Meer and Shreen Gul and Daniel Espinosa Gonzalez and Xavier Alapont and Alex Hoover and Gunjan Chhablani and Freddie Vargus and Arunim Agarwal and Yibo Jiang and Deepakkumar Patil and David Outevsky and Kevin Joseph Scaria and Rajat Maheshwari and Abdelkader Dendane and Priti Shukla and Ashley Cartwright and Sergei Bogdanov and Niels Mündler and Sören Möller and Luca Arnaboldi and Kunvar Thaman and Muhammad Rehan Siddiqi and Prajvi Saxena and Himanshu Gupta and Tony Fruhauff and Glen Sherman and Mátyás Vincze and Siranut Usawasutsakorn and Dylan Ler and Anil Radhakrishnan and Innocent Enyekwe and Sk Md Salauddin and Jiang Muzhen and Aleksandr Maksapetyan and Vivien Rossbach and Chris Harjadi and Mohsen Bahaloohoreh and Claire Sparrow and Jasdeep Sidhu and Sam Ali and Song Bian and John Lai and Eric Singer and Justine Leon Uro and Greg Bateman and Mohamed Sayed and Ahmed Menshawy and Darling Duclosel and Dario Bezzi and Yashaswini Jain and Ashley Aaron and Murat Tiryakioglu and Sheeshram Siddh and Keith Krenek and Imad Ali Shah and Jun Jin and Scott Creighton and Denis Peskoff and Zienab EL-Wasif and Ragavendran P and Michael Richmond and Joseph McGowan and Tejal Patwardhan and Hao-Yu Sun and Ting Sun and Nikola Zubić and Samuele Sala and Stephen Ebert and Jean Kaddour and Manuel Schottdorf and Dianzhuo Wang and Gerol Petruzella and Alex Meiburg and Tilen Medved and Ali ElSheikh and S Ashwin Hebbar and Lorenzo Vaquero and Xianjun Yang and Jason Poulos and Vilém Zouhar and Sergey Bogdanik and Mingfang Zhang and Jorge Sanz-Ros and David Anugraha and Yinwei Dai and Anh N. Nhu and Xue Wang and Ali Anil Demircali and Zhibai Jia and Yuyin Zhou and Juncheng Wu and Mike He and Nitin Chandok and Aarush Sinha and Gaoxiang Luo and Long Le and Mickaël Noyé and Michał Perełkiewicz and Ioannis Pantidis and Tianbo Qi and Soham Sachin Purohit and Letitia Parcalabescu and Thai-Hoa Nguyen and Genta Indra Winata and Edoardo M. Ponti and Hanchen Li and Kaustubh Dhole and Jongee Park and Dario Abbondanza and Yuanli Wang and Anupam Nayak and Diogo M. Caetano and Antonio A. W. L. Wong and Maria del Rio-Chanona and Dániel Kondor and Pieter Francois and Ed Chalstrey and Jakob Zsambok and Dan Hoyer and Jenny Reddish and Jakob Hauser and Francisco-Javier Rodrigo-Ginés and Suchandra Datta and Maxwell Shepherd and Thom Kamphuis and Qizheng Zhang and Hyunjun Kim and Ruiji Sun and Jianzhu Yao and Franck Dernoncourt and Satyapriya Krishna and Sina Rismanchian and Bonan Pu and Francesco Pinto and Yingheng Wang and Kumar Shridhar and Kalon J. Overholt and Glib Briia and Hieu Nguyen and  David and Soler Bartomeu and Tony CY Pang and Adam Wecker and Yifan Xiong and Fanfei Li and Lukas S. Huber and Joshua Jaeger and Romano De Maddalena and Xing Han Lù and Yuhui Zhang and Claas Beger and Patrick Tser Jern Kon and Sean Li and Vivek Sanker and Ming Yin and Yihao Liang and Xinlu Zhang and Ankit Agrawal and Li S. Yifei and Zechen Zhang and Mu Cai and Yasin Sonmez and Costin Cozianu and Changhao Li and Alex Slen and Shoubin Yu and Hyun Kyu Park and Gabriele Sarti and Marcin Briański and Alessandro Stolfo and Truong An Nguyen and Mike Zhang and Yotam Perlitz and Jose Hernandez-Orallo and Runjia Li and Amin Shabani and Felix Juefei-Xu and Shikhar Dhingra and Orr Zohar and My Chiffon Nguyen and Alexander Pondaven and Abdurrahim Yilmaz and Xuandong Zhao and Chuanyang Jin and Muyan Jiang and Stefan Todoran and Xinyao Han and Jules Kreuer and Brian Rabern and Anna Plassart and Martino Maggetti and Luther Yap and Robert Geirhos and Jonathon Kean and Dingsu Wang and Sina Mollaei and Chenkai Sun and Yifan Yin and Shiqi Wang and Rui Li and Yaowen Chang and Anjiang Wei and Alice Bizeul and Xiaohan Wang and Alexandre Oliveira Arrais and Kushin Mukherjee and Jorge Chamorro-Padial and Jiachen Liu and Xingyu Qu and Junyi Guan and Adam Bouyamourn and Shuyu Wu and Martyna Plomecka and Junda Chen and Mengze Tang and Jiaqi Deng and Shreyas Subramanian and Haocheng Xi and Haoxuan Chen and Weizhi Zhang and Yinuo Ren and Haoqin Tu and Sejong Kim and Yushun Chen and Sara Vera Marjanović and Junwoo Ha and Grzegorz Luczyna and Jeff J. Ma and Zewen Shen and Dawn Song and Cedegao E. Zhang and Zhun Wang and Gaël Gendron and Yunze Xiao and Leo Smucker and Erica Weng and Kwok Hao Lee and Zhe Ye and Stefano Ermon and Ignacio D. Lopez-Miguel and Theo Knights and Anthony Gitter and Namkyu Park and Boyi Wei and Hongzheng Chen and Kunal Pai and Ahmed Elkhanany and Han Lin and Philipp D. Siedler and Jichao Fang and Ritwik Mishra and Károly Zsolnai-Fehér and Xilin Jiang and Shadab Khan and Jun Yuan and Rishab Kumar Jain and Xi Lin and Mike Peterson and Zhe Wang and Aditya Malusare and Maosen Tang and Isha Gupta and Ivan Fosin and Timothy Kang and Barbara Dworakowska and Kazuki Matsumoto and Guangyao Zheng and Gerben Sewuster and Jorge Pretel Villanueva and Ivan Rannev and Igor Chernyavsky and Jiale Chen and Deepayan Banik and Ben Racz and Wenchao Dong and Jianxin Wang and Laila Bashmal and Duarte V. Gonçalves and Wei Hu and Kaushik Bar and Ondrej Bohdal and Atharv Singh Patlan and Shehzaad Dhuliawala and Caroline Geirhos and Julien Wist and Yuval Kansal and Bingsen Chen and Kutay Tire and Atak Talay Yücel and Brandon Christof and Veerupaksh Singla and Zijian Song and Sanxing Chen and Jiaxin Ge and Kaustubh Ponkshe and Isaac Park and Tianneng Shi and Martin Q. Ma and Joshua Mak and Sherwin Lai and Antoine Moulin and Zhuo Cheng and Zhanda Zhu and Ziyi Zhang and Vaidehi Patil and Ketan Jha and Qiutong Men and Jiaxuan Wu and Tianchi Zhang and Bruno Hebling Vieira and Alham Fikri Aji and Jae-Won Chung and Mohammed Mahfoud and Ha Thi Hoang and Marc Sperzel and Wei Hao and Kristof Meding and Sihan Xu and Vassilis Kostakos and Davide Manini and Yueying Liu and Christopher Toukmaji and Jay Paek and Eunmi Yu and Arif Engin Demircali and Zhiyi Sun and Ivan Dewerpe and Hongsen Qin and Roman Pflugfelder and James Bailey and Johnathan Morris and Ville Heilala and Sybille Rosset and Zishun Yu and Peter E. Chen and Woongyeong Yeo and Eeshaan Jain and Ryan Yang and Sreekar Chigurupati and Julia Chernyavsky and Sai Prajwal Reddy and Subhashini Venugopalan and Hunar Batra and Core Francisco Park and Hieu Tran and Guilherme Maximiano and Genghan Zhang and Yizhuo Liang and Hu Shiyu and Rongwu Xu and Rui Pan and Siddharth Suresh and Ziqi Liu and Samaksh Gulati and Songyang Zhang and Peter Turchin and Christopher W. Bartlett and Christopher R. Scotese and Phuong M. Cao and Ben Wu and Jacek Karwowski and Davide Scaramuzza and Aakaash Nattanmai and Gordon McKellips and Anish Cheraku and Asim Suhail and Ethan Luo and Marvin Deng and Jason Luo and Ashley Zhang and Kavin Jindel and Jay Paek and Kasper Halevy and Allen Baranov and Michael Liu and Advaith Avadhanam and David Zhang and Vincent Cheng and Brad Ma and Evan Fu and Liam Do and Joshua Lass and Hubert Yang and Surya Sunkari and Vishruth Bharath and Violet Ai and James Leung and Rishit Agrawal and Alan Zhou and Kevin Chen and Tejas Kalpathi and Ziqi Xu and Gavin Wang and Tyler Xiao and Erik Maung and Sam Lee and Ryan Yang and Roy Yue and Ben Zhao and Julia Yoon and Sunny Sun and Aryan Singh and Ethan Luo and Clark Peng and Tyler Osbey and Taozhi Wang and Daryl Echeazu and Hubert Yang and Timothy Wu and Spandan Patel and Vidhi Kulkarni and Vijaykaarti Sundarapandiyan and Ashley Zhang and Andrew Le and Zafir Nasim and Srikar Yalam and Ritesh Kasamsetty and Soham Samal and Hubert Yang and David Sun and Nihar Shah and Abhijeet Saha and Alex Zhang and Leon Nguyen and Laasya Nagumalli and Kaixin Wang and Alan Zhou and Aidan Wu and Jason Luo and Anwith Telluri and Steven Dillmann and Zhengxiang Wang and Junyu Luo and Hugo Lunn and Artem Gazizov and Haitz Sáez de Ocáriz Borde and Ivan Trus and Morgan Hervault and Zheyu Zhang and Bo Chen and Yuchen Wu and Christopher J. Cordier and Gün Kaynar and Cansin Ayvaz and Polina Avdiunina and Johannes Brust and Xingjian Diao and K. D. Meaney and Yifan Gu and Chenyu Wang and Chenzhuo Dong and William Wright and Simon Brave and Owen Root and Jiayuan Liu and Chow Chun Lok and Tianqin Li and Shiyi Du and Dailan He and Lufeiya Liu and Sina Jamalzadegan and Anil Ramakrishna and Xuanqing Xu and Xin Qing and Xin Luo and Wenkai Li and Shi Bo and Filipp Gusev and Maximos Skandalis and Desheng Ma and Chunhui Zhang and Haoran Qiu and Allen G Hart and Rickard Brüel Gabrielsson and Ido Akov and Artem Lukoianov and Summer Yue and Alexandr Wang and Dan Hendrycks},
  title     = {Humanity's Last Exam},
  journal   = {arXiv preprint arXiv:2501.14249},
  year      = {2025},
  eprint    = {2501.14249},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2501.14249},
}

@article{glazer2024frontiermath,
  author    = {Elliot Glazer and Ege Erdil and Tamay Besiroglu and Diego Chicharro and Evan Chen and Alex Gunning and Caroline Falkman Olsson and Jean-Stanislas Denain and Anson Ho and Emily de Oliveira Santos and Olli Järviniemi and Matthew Barnett and Robert Sandler and Matej Vrzala and Jaime Sevilla and Qiuyu Ren and Elizabeth Pratt and Lionel Levine and Grant Barkley and Natalie Stewart and Bogdan Grechuk and Tetiana Grechuk and Shreepranav Varma Enugandla and Mark Wildon},
  title     = {FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI},
  journal   = {arXiv preprint arXiv:2411.04872},
  year      = {2024},
  eprint    = {2411.04872},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2411.04872},
}

@inproceedings{zheng2023mcqrobust,
  author    = {Chujie Zheng and Hao Zhou and Fandong Meng and Jie Zhou and Minlie Huang},
  title     = {Large Language Models Are Not Robust Multiple Choice Selectors},
  booktitle = {ICLR 2024},
  year      = {2023},
  eprint    = {2309.03882},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2309.03882},
}

@inproceedings{sclar2023promptsensitivity,
  author    = {Melanie Sclar and Yejin Choi and Yulia Tsvetkov and Alane Suhr},
  title     = {Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting},
  booktitle = {ICLR 2024},
  year      = {2023},
  eprint    = {2310.11324},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2310.11324},
}

@article{dehghani2021benchmarklottery,
  author    = {Mostafa Dehghani and Yi Tay and Alexey A. Gritsenko and Zhe Zhao and Neil Houlsby and Fernando Diaz and Donald Metzler and Oriol Vinyals},
  title     = {The Benchmark Lottery},
  journal   = {arXiv preprint arXiv:2107.07002},
  year      = {2021},
  eprint    = {2107.07002},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2107.07002},
}

@article{kapoor2022leakage,
  author    = {Sayash Kapoor and Arvind Narayanan},
  title     = {Leakage and the Reproducibility Crisis in ML-based Science},
  journal   = {arXiv preprint arXiv:2207.07048},
  year      = {2022},
  eprint    = {2207.07048},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2207.07048},
}

@article{damour2020underspecification,
  author    = {Alexander D'Amour and Katherine Heller and Dan Moldovan and Ben Adlam and Babak Alipanahi and Alex Beutel and Christina Chen and Jonathan Deaton and Jacob Eisenstein and Matthew D. Hoffman and Farhad Hormozdiari and Neil Houlsby and Shaobo Hou and Ghassen Jerfel and Alan Karthikesalingam and Mario Lucic and Yian Ma and Cory McLean and Diana Mincu and Akinori Mitani and Andrea Montanari and Zachary Nado and Vivek Natarajan and Christopher Nielson and Thomas F. Osborne and Rajiv Raman and Kim Ramasamy and Rory Sayres and Jessica Schrouff and Martin Seneviratne and Shannon Sequeira and Harini Suresh and Victor Veitch and Max Vladymyrov and Xuezhi Wang and Kellie Webster and Steve Yadlowsky and Taedong Yun and Xiaohua Zhai and D. Sculley},
  title     = {Underspecification Presents Challenges for Credibility in Modern Machine Learning},
  journal   = {arXiv preprint arXiv:2011.03395},
  year      = {2020},
  eprint    = {2011.03395},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2011.03395},
}

@article{geirhos2020shortcut,
  author    = {Robert Geirhos and Jörn-Henrik Jacobsen and Claudio Michaelis and Richard Zemel and Wieland Brendel and Matthias Bethge and Felix A. Wichmann},
  title     = {Shortcut Learning in Deep Neural Networks},
  journal   = {arXiv preprint arXiv:2004.07780},
  year      = {2020},
  eprint    = {2004.07780},
  archivePrefix = {arXiv},
  primaryClass = {cs.CV},
  url       = {https://arxiv.org/abs/2004.07780},
}

@article{miller2020distributionshift,
  author    = {John Miller and Karl Krauth and Benjamin Recht and Ludwig Schmidt},
  title     = {The Effect of Natural Distribution Shift on Question Answering Models},
  journal   = {arXiv preprint arXiv:2004.14444},
  year      = {2020},
  eprint    = {2004.14444},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2004.14444},
}

@inproceedings{kaushik2020counterfactual,
  author    = {Divyansh Kaushik and Eduard Hovy and Zachary C. Lipton},
  title     = {Learning the Difference that Makes a Difference with Counterfactually-Augmented Data},
  booktitle = {ICLR 2020},
  year      = {2019},
  eprint    = {1909.12434},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/1909.12434},
}

@inproceedings{ribeiro2020checklist,
  author    = {Marco Tulio Ribeiro and Tongshuang Wu and Carlos Guestrin and Sameer Singh},
  title     = {Beyond Accuracy: Behavioral Testing of NLP models with CheckList},
  booktitle = {Association for Computational Linguistics (ACL), 2020},
  year      = {2020},
  eprint    = {2005.04118},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2005.04118},
}

@article{schaeffer2023mirage,
  author    = {Rylan Schaeffer and Brando Miranda and Sanmi Koyejo},
  title     = {Are Emergent Abilities of Large Language Models a Mirage?},
  journal   = {arXiv preprint arXiv:2304.15004},
  year      = {2023},
  eprint    = {2304.15004},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2304.15004},
}

@article{wei2022cot,
  author    = {Jason Wei and Xuezhi Wang and Dale Schuurmans and Maarten Bosma and Brian Ichter and Fei Xia and Ed Chi and Quoc Le and Denny Zhou},
  title     = {Chain-of-Thought Prompting Elicits Reasoning in Large Language Models},
  journal   = {arXiv preprint arXiv:2201.11903},
  year      = {2022},
  eprint    = {2201.11903},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2201.11903},
}

@inproceedings{wang2022selfconsistency,
  author    = {Xuezhi Wang and Jason Wei and Dale Schuurmans and Quoc Le and Ed Chi and Sharan Narang and Aakanksha Chowdhery and Denny Zhou},
  title     = {Self-Consistency Improves Chain of Thought Reasoning in Language Models},
  booktitle = {ICLR 2023},
  year      = {2022},
  eprint    = {2203.11171},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2203.11171},
}

@article{snell2024testtime,
  author    = {Charlie Snell and Jaehoon Lee and Kelvin Xu and Aviral Kumar},
  title     = {Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters},
  journal   = {arXiv preprint arXiv:2408.03314},
  year      = {2024},
  eprint    = {2408.03314},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2408.03314},
}

@article{muennighoff2025s1,
  author    = {Niklas Muennighoff and Zitong Yang and Weijia Shi and Xiang Lisa Li and Li Fei-Fei and Hannaneh Hajishirzi and Luke Zettlemoyer and Percy Liang and Emmanuel Candès and Tatsunori Hashimoto},
  title     = {s1: Simple test-time scaling},
  journal   = {arXiv preprint arXiv:2501.19393},
  year      = {2025},
  eprint    = {2501.19393},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2501.19393},
}

@article{guo2025deepseekr1,
  author    = { DeepSeek-AI and Daya Guo and Dejian Yang and Haowei Zhang and Junxiao Song and Peiyi Wang and Qihao Zhu and Runxin Xu and Ruoyu Zhang and Shirong Ma and Xiao Bi and Xiaokang Zhang and Xingkai Yu and Yu Wu and Z. F. Wu and Zhibin Gou and Zhihong Shao and Zhuoshu Li and Ziyi Gao and Aixin Liu and Bing Xue and Bingxuan Wang and Bochao Wu and Bei Feng and Chengda Lu and Chenggang Zhao and Chengqi Deng and Chenyu Zhang and Chong Ruan and Damai Dai and Deli Chen and Dongjie Ji and Erhang Li and Fangyun Lin and Fucong Dai and Fuli Luo and Guangbo Hao and Guanting Chen and Guowei Li and H. Zhang and Han Bao and Hanwei Xu and Haocheng Wang and Honghui Ding and Huajian Xin and Huazuo Gao and Hui Qu and Hui Li and Jianzhong Guo and Jiashi Li and Jiawei Wang and Jingchang Chen and Jingyang Yuan and Junjie Qiu and Junlong Li and J. L. Cai and Jiaqi Ni and Jian Liang and Jin Chen and Kai Dong and Kai Hu and Kaige Gao and Kang Guan and Kexin Huang and Kuai Yu and Lean Wang and Lecong Zhang and Liang Zhao and Litong Wang and Liyue Zhang and Lei Xu and Leyi Xia and Mingchuan Zhang and Minghua Zhang and Minghui Tang and Meng Li and Miaojun Wang and Mingming Li and Ning Tian and Panpan Huang and Peng Zhang and Qiancheng Wang and Qinyu Chen and Qiushi Du and Ruiqi Ge and Ruisong Zhang and Ruizhe Pan and Runji Wang and R. J. Chen and R. L. Jin and Ruyi Chen and Shanghao Lu and Shangyan Zhou and Shanhuang Chen and Shengfeng Ye and Shiyu Wang and Shuiping Yu and Shunfeng Zhou and Shuting Pan and S. S. Li and Shuang Zhou and Shaoqing Wu and Shengfeng Ye and Tao Yun and Tian Pei and Tianyu Sun and T. Wang and Wangding Zeng and Wanjia Zhao and Wen Liu and Wenfeng Liang and Wenjun Gao and Wenqin Yu and Wentao Zhang and W. L. Xiao and Wei An and Xiaodong Liu and Xiaohan Wang and Xiaokang Chen and Xiaotao Nie and Xin Cheng and Xin Liu and Xin Xie and Xingchao Liu and Xinyu Yang and Xinyuan Li and Xuecheng Su and Xuheng Lin and X. Q. Li and Xiangyue Jin and Xiaojin Shen and Xiaosha Chen and Xiaowen Sun and Xiaoxiang Wang and Xinnan Song and Xinyi Zhou and Xianzu Wang and Xinxia Shan and Y. K. Li and Y. Q. Wang and Y. X. Wei and Yang Zhang and Yanhong Xu and Yao Li and Yao Zhao and Yaofeng Sun and Yaohui Wang and Yi Yu and Yichao Zhang and Yifan Shi and Yiliang Xiong and Ying He and Yishi Piao and Yisong Wang and Yixuan Tan and Yiyang Ma and Yiyuan Liu and Yongqiang Guo and Yuan Ou and Yuduan Wang and Yue Gong and Yuheng Zou and Yujia He and Yunfan Xiong and Yuxiang Luo and Yuxiang You and Yuxuan Liu and Yuyang Zhou and Y. X. Zhu and Yanhong Xu and Yanping Huang and Yaohui Li and Yi Zheng and Yuchen Zhu and Yunxian Ma and Ying Tang and Yukun Zha and Yuting Yan and Z. Z. Ren and Zehui Ren and Zhangli Sha and Zhe Fu and Zhean Xu and Zhenda Xie and Zhengyan Zhang and Zhewen Hao and Zhicheng Ma and Zhigang Yan and Zhiyu Wu and Zihui Gu and Zijia Zhu and Zijun Liu and Zilin Li and Ziwei Xie and Ziyang Song and Zizheng Pan and Zhen Huang and Zhipeng Xu and Zhongyu Zhang and Zhen Zhang},
  title     = {DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning},
  journal   = {Nature volume 645, pages 633-638 (2025)},
  year      = {2025},
  eprint    = {2501.12948},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2501.12948},
}

@article{chen2024overthinking,
  author    = {Xingyu Chen and Jiahao Xu and Tian Liang and Zhiwei He and Jianhui Pang and Dian Yu and Linfeng Song and Qiuzhi Liu and Mengfei Zhou and Zhuosheng Zhang and Rui Wang and Zhaopeng Tu and Haitao Mi and Dong Yu},
  title     = {Do NOT Think That Much for 2+3=? On the Overthinking of o1-Like LLMs},
  journal   = {arXiv preprint arXiv:2412.21187},
  year      = {2024},
  eprint    = {2412.21187},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2412.21187},
}

@article{brown2024monkeys,
  author    = {Bradley Brown and Jordan Juravsky and Ryan Ehrlich and Ronald Clark and Quoc V. Le and Christopher Ré and Azalia Mirhoseini},
  title     = {Large Language Monkeys: Scaling Inference Compute with Repeated Sampling},
  journal   = {arXiv preprint arXiv:2407.21787},
  year      = {2024},
  eprint    = {2407.21787},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2407.21787},
}

@inproceedings{turpin2023faithfulness,
  author    = {Miles Turpin and Julian Michael and Ethan Perez and Samuel R. Bowman},
  title     = {Language Models Don't Always Say What They Think: Unfaithful Explanations in Chain-of-Thought Prompting},
  booktitle = {NeurIPS 2023},
  year      = {2023},
  eprint    = {2305.04388},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2305.04388},
}

@article{lanham2023measuring,
  author    = {Tamera Lanham and Anna Chen and Ansh Radhakrishnan and Benoit Steiner and Carson Denison and Danny Hernandez and Dustin Li and Esin Durmus and Evan Hubinger and Jackson Kernion and Kamilė Lukošiūtė and Karina Nguyen and Newton Cheng and Nicholas Joseph and Nicholas Schiefer and Oliver Rausch and Robin Larson and Sam McCandlish and Sandipan Kundu and Saurav Kadavath and Shannon Yang and Thomas Henighan and Timothy Maxwell and Timothy Telleen-Lawton and Tristan Hume and Zac Hatfield-Dodds and Jared Kaplan and Jan Brauner and Samuel R. Bowman and Ethan Perez},
  title     = {Measuring Faithfulness in Chain-of-Thought Reasoning},
  journal   = {arXiv preprint arXiv:2307.13702},
  year      = {2023},
  eprint    = {2307.13702},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2307.13702},
}

@article{brown2020gpt3,
  author    = {Tom B. Brown and Benjamin Mann and Nick Ryder and Melanie Subbiah and Jared Kaplan and Prafulla Dhariwal and Arvind Neelakantan and Pranav Shyam and Girish Sastry and Amanda Askell and Sandhini Agarwal and Ariel Herbert-Voss and Gretchen Krueger and Tom Henighan and Rewon Child and Aditya Ramesh and Daniel M. Ziegler and Jeffrey Wu and Clemens Winter and Christopher Hesse and Mark Chen and Eric Sigler and Mateusz Litwin and Scott Gray and Benjamin Chess and Jack Clark and Christopher Berner and Sam McCandlish and Alec Radford and Ilya Sutskever and Dario Amodei},
  title     = {Language Models are Few-Shot Learners},
  journal   = {arXiv preprint arXiv:2005.14165},
  year      = {2020},
  eprint    = {2005.14165},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2005.14165},
}

@inproceedings{min2022rethinking,
  author    = {Sewon Min and Xinxi Lyu and Ari Holtzman and Mikel Artetxe and Mike Lewis and Hannaneh Hajishirzi and Luke Zettlemoyer},
  title     = {Rethinking the Role of Demonstrations: What Makes In-Context Learning Work?},
  booktitle = {EMNLP 2022},
  year      = {2022},
  eprint    = {2202.12837},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2202.12837},
  note      = {17 pages; 12 figures. Code available at https://github.com/Alrope123/rethinking-demonst…},
}

@inproceedings{chan2022datadistributional,
  author    = {Stephanie C. Y. Chan and Adam Santoro and Andrew K. Lampinen and Jane X. Wang and Aaditya Singh and Pierre H. Richemond and Jay McClelland and Felix Hill},
  title     = {Data Distributional Properties Drive Emergent In-Context Learning in Transformers},
  booktitle = {NeurIPS 2022},
  year      = {2022},
  eprint    = {2205.05055},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2205.05055},
  note      = {Code is available at: https://github.com/deepmind/emergent_in_context_learning},
}

@article{liu2022whatmakesgood,
  author    = {Jiachang Liu and Dinghan Shen and Yizhe Zhang and Bill Dolan and Lawrence Carin and Weizhu Chen},
  title     = {What Makes Good In-Context Examples for GPT-$3$?},
  journal   = {arXiv preprint arXiv:2101.06804},
  year      = {2021},
  eprint    = {2101.06804},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2101.06804},
}

@article{shinn2023reflexion,
  author    = {Noah Shinn and Federico Cassano and Edward Berman and Ashwin Gopinath and Karthik Narasimhan and Shunyu Yao},
  title     = {Reflexion: Language Agents with Verbal Reinforcement Learning},
  journal   = {arXiv preprint arXiv:2303.11366},
  year      = {2023},
  eprint    = {2303.11366},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2303.11366},
}

@article{zhao2023expel,
  author    = {Andrew Zhao and Daniel Huang and Quentin Xu and Matthieu Lin and Yong-Jin Liu and Gao Huang},
  title     = {ExpeL: LLM Agents Are Experiential Learners},
  journal   = {arXiv preprint arXiv:2308.10144},
  year      = {2023},
  eprint    = {2308.10144},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2308.10144},
}

@article{wang2023voyager,
  author    = {Guanzhi Wang and Yuqi Xie and Yunfan Jiang and Ajay Mandlekar and Chaowei Xiao and Yuke Zhu and Linxi Fan and Anima Anandkumar},
  title     = {Voyager: An Open-Ended Embodied Agent with Large Language Models},
  journal   = {arXiv preprint arXiv:2305.16291},
  year      = {2023},
  eprint    = {2305.16291},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2305.16291},
}

@inproceedings{zhang2025ace,
  author    = {Qizheng Zhang and Changran Hu and Shubhangi Upasani and Boyuan Ma and Fenglu Hong and Vamsidhar Kamanuru and Jay Rainton and Chen Wu and Mengmeng Ji and Hanchen Li and Urmish Thakker and James Zou and Kunle Olukotun},
  title     = {Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models},
  booktitle = {ICLR 2026},
  year      = {2025},
  eprint    = {2510.04618},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2510.04618},
}

@inproceedings{evolver2025,
  author    = {Rong Wu and Xiaoman Wang and Jianbiao Mei and Pinlong Cai and Daocheng Fu and Cheng Yang and Licheng Wen and Xuemeng Yang and Yufan Shen and Yuxin Wang and Botian Shi},
  title     = {EvolveR: Self-Evolving LLM Agents through an Experience-Driven Lifecycle},
  booktitle = {ICML 2026},
  year      = {2025},
  eprint    = {2510.16079},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2510.16079},
}

@inproceedings{wang2025amem,
  author    = {Wujiang Xu and Zujie Liang and Kai Mei and Hang Gao and Juntao Tan and Yongfeng Zhang},
  title     = {A-MEM: Agentic Memory for LLM Agents},
  booktitle = {NeurIPS 2025},
  year      = {2025},
  eprint    = {2502.12110},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2502.12110},
}

@article{zhong2023memorybank,
  author    = {Wanjun Zhong and Lianghong Guo and Qiqi Gao and He Ye and Yanlin Wang},
  title     = {MemoryBank: Enhancing Large Language Models with Long-Term Memory},
  journal   = {arXiv preprint arXiv:2305.10250},
  year      = {2023},
  eprint    = {2305.10250},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2305.10250},
}

@article{yuan2026skillopt,
  author    = {Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
  title     = {SkillOpt: Executive Strategy for Self-Evolving Agent Skills},
  journal   = {arXiv preprint arXiv:2605.23904},
  year      = {2026},
  eprint    = {2605.23904},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2605.23904},
}

@article{ai4aittesttime2026,
  author    = {Cheng Qian and Wenting Zhao and Liangwei Yang and Heng Wang and Jielin Qiu and Heng Ji and Silvio Savarese and Huan Wang and Shelby Heinecke},
  title     = {AI4AI at Test-Time: Strong-to-Weak Capability Transfer via Harnesses},
  journal   = {arXiv preprint arXiv:2608.12307},
  year      = {2026},
  eprint    = {2608.12307},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2608.12307},
}

@article{yuksekgonul2024textgrad,
  author    = {Mert Yuksekgonul and Federico Bianchi and Joseph Boen and Sheng Liu and Zhi Huang and Carlos Guestrin and James Zou},
  title     = {TextGrad: Automatic "Differentiation" via Text},
  journal   = {arXiv preprint arXiv:2406.07496},
  year      = {2024},
  eprint    = {2406.07496},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2406.07496},
}

@article{khattab2023dspy,
  author    = {Omar Khattab and Arnav Singhvi and Paridhi Maheshwari and Zhiyuan Zhang and Keshav Santhanam and Sri Vardhamanan and Saiful Haq and Ashutosh Sharma and Thomas T. Joshi and Hanna Moazam and Heather Miller and Matei Zaharia and Christopher Potts},
  title     = {DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines},
  journal   = {arXiv preprint arXiv:2310.03714},
  year      = {2023},
  eprint    = {2310.03714},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2310.03714},
}

@article{fernando2023promptbreeder,
  author    = {Chrisantha Fernando and Dylan Banarse and Henryk Michalewski and Simon Osindero and Tim Rocktäschel},
  title     = {Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution},
  journal   = {arXiv preprint arXiv:2309.16797},
  year      = {2023},
  eprint    = {2309.16797},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2309.16797},
}

@inproceedings{yuan2024selfrewarding,
  author    = {Weizhe Yuan and Richard Yuanzhe Pang and Kyunghyun Cho and Xian Li and Sainbayar Sukhbaatar and Jing Xu and Jason Weston},
  title     = {Self-Rewarding Language Models},
  booktitle = {ICML 2024},
  year      = {2024},
  eprint    = {2401.10020},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2401.10020},
}

@article{zelikman2022star,
  author    = {Eric Zelikman and Yuhuai Wu and Jesse Mu and Noah D. Goodman},
  title     = {STaR: Bootstrapping Reasoning With Reasoning},
  journal   = {arXiv preprint arXiv:2203.14465},
  year      = {2022},
  eprint    = {2203.14465},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2203.14465},
}

@article{zelikman2024quietstar,
  author    = {Eric Zelikman and Georges Harik and Yijia Shao and Varuna Jayasiri and Nick Haber and Noah D. Goodman},
  title     = {Quiet-STaR: Language Models Can Teach Themselves to Think Before Speaking},
  journal   = {arXiv preprint arXiv:2403.09629},
  year      = {2024},
  eprint    = {2403.09629},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2403.09629},
}

@article{chen2024lats,
  author    = {Andy Zhou and Kai Yan and Michal Shlapentokh-Rothman and Haohan Wang and Yu-Xiong Wang},
  title     = {Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models},
  journal   = {arXiv preprint arXiv:2310.04406},
  year      = {2023},
  eprint    = {2310.04406},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2310.04406},
}

@article{wu2024metarewarding,
  author    = {Tianhao Wu and Weizhe Yuan and Olga Golovneva and Jing Xu and Yuandong Tian and Jiantao Jiao and Jason Weston and Sainbayar Sukhbaatar},
  title     = {Meta-Rewarding Language Models: Self-Improving Alignment with LLM-as-a-Meta-Judge},
  journal   = {arXiv preprint arXiv:2407.19594},
  year      = {2024},
  eprint    = {2407.19594},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2407.19594},
}

@article{singh2023restem,
  author    = {Avi Singh and John D. Co-Reyes and Rishabh Agarwal and Ankesh Anand and Piyush Patil and Xavier Garcia and Peter J. Liu and James Harrison and Jaehoon Lee and Kelvin Xu and Aaron Parisi and Abhishek Kumar and Alex Alemi and Alex Rizkowsky and Azade Nova and Ben Adlam and Bernd Bohnet and Gamaleldin Elsayed and Hanie Sedghi and Igor Mordatch and Isabelle Simpson and Izzeddin Gur and Jasper Snoek and Jeffrey Pennington and Jiri Hron and Kathleen Kenealy and Kevin Swersky and Kshiteej Mahajan and Laura Culp and Lechao Xiao and Maxwell L. Bileschi and Noah Constant and Roman Novak and Rosanne Liu and Tris Warkentin and Yundi Qian and Yamini Bansal and Ethan Dyer and Behnam Neyshabur and Jascha Sohl-Dickstein and Noah Fiedel},
  title     = {Beyond Human Data: Scaling Self-Training for Problem-Solving with Language Models},
  journal   = {arXiv preprint arXiv:2312.06585},
  year      = {2023},
  eprint    = {2312.06585},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2312.06585},
}

@article{huang2022selfimprove,
  author    = {Jiaxin Huang and Shixiang Shane Gu and Le Hou and Yuexin Wu and Xuezhi Wang and Hongkun Yu and Jiawei Han},
  title     = {Large Language Models Can Self-Improve},
  journal   = {arXiv preprint arXiv:2210.11610},
  year      = {2022},
  eprint    = {2210.11610},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2210.11610},
}

@inproceedings{wang2022selfinstruct,
  author    = {Yizhong Wang and Yeganeh Kordi and Swaroop Mishra and Alisa Liu and Noah A. Smith and Daniel Khashabi and Hannaneh Hajishirzi},
  title     = {Self-Instruct: Aligning Language Models with Self-Generated Instructions},
  booktitle = {ACL 2023},
  year      = {2022},
  eprint    = {2212.10560},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2212.10560},
}

@inproceedings{lee2023rlaif,
  author    = {Harrison Lee and Samrat Phatale and Hassan Mansoor and Thomas Mesnard and Johan Ferret and Kellie Lu and Colton Bishop and Ethan Hall and Victor Carbune and Abhinav Rastogi and Sushant Prakash},
  title     = {RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback},
  booktitle = {Proceedings of the 41st International Conference on Machine Learning, PMLR 235:26874-26901, 2024},
  year      = {2023},
  eprint    = {2309.00267},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2309.00267},
}

@inproceedings{chen2024spin,
  author    = {Zixiang Chen and Yihe Deng and Huizhuo Yuan and Kaixuan Ji and Quanquan Gu},
  title     = {Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models},
  booktitle = {ICML 2024},
  year      = {2024},
  eprint    = {2401.01335},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2401.01335},
}

@article{zweiger2025seal,
  author    = {Adam Zweiger and Jyothish Pari and Han Guo and Ekin Akyürek and Yoon Kim and Pulkit Agrawal},
  title     = {Self-Adapting Language Models},
  journal   = {arXiv preprint arXiv:2506.10943},
  year      = {2025},
  eprint    = {2506.10943},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2506.10943},
}

@article{ye2024sharpening,
  author    = {Audrey Huang and Adam Block and Dylan J. Foster and Dhruv Rohatgi and Cyril Zhang and Max Simchowitz and Jordan T. Ash and Akshay Krishnamurthy},
  title     = {Self-Improvement in Language Models: The Sharpening Mechanism},
  journal   = {arXiv preprint arXiv:2412.01951},
  year      = {2024},
  eprint    = {2412.01951},
  archivePrefix = {arXiv},
  primaryClass = {cs.AI},
  url       = {https://arxiv.org/abs/2412.01951},
}

@inproceedings{lewis2020rag,
  author    = {Patrick Lewis and Ethan Perez and Aleksandra Piktus and Fabio Petroni and Vladimir Karpukhin and Naman Goyal and Heinrich Küttler and Mike Lewis and Wen-tau Yih and Tim Rocktäschel and Sebastian Riedel and Douwe Kiela},
  title     = {Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks},
  booktitle = {NeurIPS 2020},
  year      = {2020},
  eprint    = {2005.11401},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2005.11401},
}

@article{kingma2014adam,
  author    = {Diederik P. Kingma and Jimmy Ba},
  title     = {Adam: A Method for Stochastic Optimization},
  journal   = {arXiv preprint arXiv:1412.6980},
  year      = {2014},
  eprint    = {1412.6980},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1412.6980},
}

@inproceedings{loshchilov2017adamw,
  author    = {Ilya Loshchilov and Frank Hutter},
  title     = {Decoupled Weight Decay Regularization},
  booktitle = {ICLR 2019},
  year      = {2017},
  eprint    = {1711.05101},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1711.05101},
}

@article{wilson2017adaptive,
  author    = {Ashia C. Wilson and Rebecca Roelofs and Mitchell Stern and Nathan Srebro and Benjamin Recht},
  title     = {The Marginal Value of Adaptive Gradient Methods in Machine Learning},
  journal   = {arXiv preprint arXiv:1705.08292},
  year      = {2017},
  eprint    = {1705.08292},
  archivePrefix = {arXiv},
  primaryClass = {stat.ML},
  url       = {https://arxiv.org/abs/1705.08292},
}

@inproceedings{smith2018dontdecay,
  author    = {Samuel L. Smith and Pieter-Jan Kindermans and Chris Ying and Quoc V. Le},
  title     = {Don't Decay the Learning Rate, Increase the Batch Size},
  booktitle = {ICLR 2018},
  year      = {2017},
  eprint    = {1711.00489},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1711.00489},
}

@article{mccandlish2018largbatch,
  author    = {Sam McCandlish and Jared Kaplan and Dario Amodei and OpenAI Dota Team},
  title     = {An Empirical Model of Large-Batch Training},
  journal   = {arXiv preprint arXiv:1812.06162},
  year      = {2018},
  eprint    = {1812.06162},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/1812.06162},
}

@article{kaplan2020scalinglaws,
  author    = {Jared Kaplan and Sam McCandlish and Tom Henighan and Tom B. Brown and Benjamin Chess and Rewon Child and Scott Gray and Alec Radford and Jeffrey Wu and Dario Amodei},
  title     = {Scaling Laws for Neural Language Models},
  journal   = {arXiv preprint arXiv:2001.08361},
  year      = {2020},
  eprint    = {2001.08361},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2001.08361},
}

@article{hoffmann2022chinchilla,
  author    = {Jordan Hoffmann and Sebastian Borgeaud and Arthur Mensch and Elena Buchatskaya and Trevor Cai and Eliza Rutherford and Diego de Las Casas and Lisa Anne Hendricks and Johannes Welbl and Aidan Clark and Tom Hennigan and Eric Noland and Katie Millican and George van den Driessche and Bogdan Damoc and Aurelia Guy and Simon Osindero and Karen Simonyan and Erich Elsen and Jack W. Rae and Oriol Vinyals and Laurent Sifre},
  title     = {Training Compute-Optimal Large Language Models},
  journal   = {arXiv preprint arXiv:2203.15556},
  year      = {2022},
  eprint    = {2203.15556},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url       = {https://arxiv.org/abs/2203.15556},
}

@article{grinsztajn2022tabular,
  author    = {Léo Grinsztajn and Edouard Oyallon and Gaël Varoquaux},
  title     = {Why do tree-based models still outperform deep learning on tabular data?},
  journal   = {arXiv preprint arXiv:2207.08815},
  year      = {2022},
  eprint    = {2207.08815},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2207.08815},
}

@article{hollmann2022tabpfn,
  author    = {Noah Hollmann and Samuel Müller and Katharina Eggensperger and Frank Hutter},
  title     = {TabPFN: A Transformer That Solves Small Tabular Classification Problems in a Second},
  journal   = {arXiv preprint arXiv:2207.01848},
  year      = {2022},
  eprint    = {2207.01848},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url       = {https://arxiv.org/abs/2207.01848},
}

@article{auer2002ucb,
  author    = {Peter Auer and Nicolò Cesa-Bianchi and Paul Fischer},
  title     = {Finite-time Analysis of the Multiarmed Bandit Problem},
  journal   = {Machine Learning, 47(2-3):235-256},
  year      = {2002},
  url       = {https://doi.org/10.1023/A:1013689704352},
}

@article{silver2017alphagozero,
  author    = {David Silver and Julian Schrittwieser and Karen Simonyan and Ioannis Antonoglou and Aja Huang and Arthur Guez and Thomas Hubert and Lucas Baker and Matthew Lai and Adrian Bolton and Yutian Chen and Timothy Lillicrap and Fan Hui and Laurent Sifre and George van den Driessche and Thore Graepel and Demis Hassabis},
  title     = {Mastering the Game of Go without Human Knowledge},
  journal   = {Nature, 550(7676):354-359},
  year      = {2017},
  url       = {https://doi.org/10.1038/nature24270},
}

@article{browne2012mcts,
  author    = {Cameron B. Browne and Edward Powley and Daniel Whitehouse and Simon M. Lucas and Peter I. Cowling and Philipp Rohlfshagen and Stephen Tavener and Diego Perez and Spyridon Samothrakis and Simon Colton},
  title     = {A Survey of Monte Carlo Tree Search Methods},
  journal   = {IEEE Transactions on Computational Intelligence and AI in Games, 4(1):1-43},
  year      = {2012},
  url       = {https://doi.org/10.1109/TCIAIG.2012.2186810},
}

@article{mcnemar1947,
  author    = {Quinn McNemar},
  title     = {Note on the Sampling Error of the Difference Between Correlated Proportions or Percentages},
  journal   = {Psychometrika, 12(2):153-157},
  year      = {1947},
  url       = {https://doi.org/10.1007/BF02295996},
}

@article{friedman2001gbm,
  author    = {Jerome H. Friedman},
  title     = {Greedy Function Approximation: A Gradient Boosting Machine},
  journal   = {The Annals of Statistics, 29(5):1189-1232},
  year      = {2001},
  url       = {https://doi.org/10.1214/aos/1013203451},
}