NVIDIA-NeMo · danecor · Mar 13, 2026 · Mar 15, 2026 · Mar 15, 2026 · Mar 16, 2026
diff --git a/README.md b/README.md
diff --git a/benchmarks/aime24/config.yaml b/benchmarks/aime24/config.yaml
@@ -2,20 +2,7 @@
 config_paths:
   - resources_servers/math_with_judge/configs/math_with_judge.yaml
 
-# We use `_inherit_from` directives to inherit from and not use the generic config above to ensure this benchmark config is isolated.
-aime24_math_with_judge_resources_server:
-  _inherit_from: math_with_judge
-
-aime24_math_with_judge_simple_agent:
-  _inherit_from: math_with_judge_simple_agent
-  responses_api_agents:
-    simple_agent:
-      resources_server:
-        name: aime24_math_with_judge_resources_server
-      datasets:
-      - name: aime24
-        type: benchmark
-        jsonl_fpath: benchmarks/aime24/data/aime24_benchmark.jsonl
-        prompt_config: benchmarks/aime24/prompts/default.yaml
-        prepare_script: benchmarks/aime24/prepare.py
-        num_repeats: 32
+# Rollout collection defaults — picked up directly by RolloutCollectionConfig
+agent_name: math_with_judge_simple_agent
+input_jsonl_fpath: benchmarks/aime24/data/aime24_validation.jsonl
+num_repeats: 32
diff --git a/benchmarks/aime24/data/aime24_benchmark_metrics.json b/benchmarks/aime24/data/aime24_benchmark_metrics.json
diff --git a/benchmarks/aime24/prepare.py b/benchmarks/aime24/prepare.py
@@ -26,7 +26,7 @@
 
 BENCHMARK_DIR = Path(__file__).parent
 DATA_DIR = BENCHMARK_DIR / "data"
-OUTPUT_FPATH = DATA_DIR / "aime24_benchmark.jsonl"
+OUTPUT_FPATH = DATA_DIR / "aime24_validation.jsonl"
 
 # HuggingFace dataset for AIME 2024
 HF_REPO_ID = "HuggingFaceH4/aime_2024"

diff --git a/benchmarks/aime24/prompts/default.yaml b/benchmarks/aime24/prompts/default.yaml
diff --git a/benchmarks/aime25/__init__.py b/benchmarks/aime25/__init__.py
diff --git a/benchmarks/aime25/config.yaml b/benchmarks/aime25/config.yaml
diff --git a/benchmarks/aime25/data/.gitignore b/benchmarks/aime25/data/.gitignore
diff --git a/benchmarks/aime25/data/aime25_benchmark_metrics.json b/benchmarks/aime25/data/aime25_benchmark_metrics.json
diff --git a/benchmarks/aime25/prepare.py b/benchmarks/aime25/prepare.py
diff --git a/benchmarks/aime25/prompts/default.yaml b/benchmarks/aime25/prompts/default.yaml
diff --git a/benchmarks/gpqa/README.md b/benchmarks/gpqa/README.md
diff --git a/benchmarks/gpqa/__init__.py b/benchmarks/gpqa/__init__.py
diff --git a/benchmarks/gpqa/config.yaml b/benchmarks/gpqa/config.yaml
diff --git a/benchmarks/gpqa/data/.gitignore b/benchmarks/gpqa/data/.gitignore
diff --git a/benchmarks/gpqa/data/gpqa_diamond_benchmark_metrics.json b/benchmarks/gpqa/data/gpqa_diamond_benchmark_metrics.json