[{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"cobench-prior-unspecified","comparability_caveats":["Prior environment details not re-established."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co-prior","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Percentage of problems solved"},"attempts_per_task":{"status":"known","value":1},"benchmark_version_id":"cobench-2-1","comparability_caveats":["Weights unchanged from earlier card; environment changed."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"known","value":"API safety filter off"},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co21-earlier","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":500},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"known","value":"Same 500 problems and evaluation code in Figure 2.3.4.1.A"},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Percentage of problems solved"},"attempts_per_task":{"status":"known","value":1},"benchmark_version_id":"cobench-2-1","comparability_caveats":["Opus 5.5 run 13 days later. One environment change estimated at 0–1 point; two unmeasured."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"known","value":"API safety filter off"},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co21-later","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":500},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"known","value":"Same 500 problems and evaluation code in Figure 2.3.4.1.A"},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean rubric reward (%)"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"openai-research-debugging-2026","comparability_caveats":["Source describes 41 bugs and six alignment-auditing tasks, but does not specify which contribute to the Figure 84 mean."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"openai-research-debugging-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.1","source_id":"src-astra"},{"locator":"Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean rubric reward"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"kernelgen-1p-2026","comparability_caveats":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"OpenAI first-party hardware; exact hardware not disclosed"},"human_intervention":{"status":"not_reported","value":null},"id":"kernelgen-1p-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"known","value":"Kernel development environment, benchmark harness, reference materials, performance and correctness tests"},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"nanogpt-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One H100 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"nanogpt-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.3","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"clamp((trial − base)/(instruct − base), 0, 1); cheating trials assigned zero"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"posttrainbench-lite-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One H100 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"posttrainbench-lite-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.4","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":12},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"5 hours"}},{"aggregate_method":{"status":"known","value":"Average pass@1"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"grb-2026-08","comparability_caveats":["Bugged tasks retained."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"grb-current","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"pass1","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Scoped internal agent scaffold"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed p.37; PDF index 36; GRB Internal Benchmark","source_id":"src-grb"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":74},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"2 hours per task"}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"grb-2026-08","comparability_caveats":["No detailed transcript analysis of older runs."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"grb-earlier","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"pass1","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed p.37; PDF index 36; Results and footnote 10","source_id":"src-grb"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-basic","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"BasicAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"12 hours"}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-iterative","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"12 hours"}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-iterative36","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"36 hours"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-o1-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":16},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-4o-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":36},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-llama-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-claude-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Percentile rank against test-time-compute reference distribution"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-revised-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"mle-revised-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"percentile","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Up to three leaderboard submissions"},"source_refs":[{"locator":"§10.1.3.5","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":72},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Bootstrap max of 16; mean for hidden-score Scaling Law task"},"attempts_per_task":{"status":"known","value":16},"benchmark_version_id":"rebench-gemini3","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"rebench-gemini-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":false},"metric_id":"assessment","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":24},"scaffold":{"status":"known","value":"METR Modular with minimal changes"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed pp.13–15; PDF indices 13–15; Machine Learning R&D","source_id":"src-gemini3"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":5},"task_exclusions":{"status":"known","value":"Finetune GPT-2 for QA; Scaffolding for Rust Codecontest"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"32 cumulative hours: 16 × 2-hour attempts"}},{"aggregate_method":{"status":"known","value":"Fitted 50% success horizon"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"th-1-0","comparability_caveats":["Version-specific task distribution; no evaluation dates supplied."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"th-1-0-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"p50","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Vivaria"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":170},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Fitted 50% success horizon"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"th-1-1","comparability_caveats":["Version-specific task distribution; no evaluation dates supplied."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"th-1-1-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"p50","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Inspect"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":228},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-opus-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-sol-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-kimi-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-sonnet-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-terra-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-luna-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Person-time-weighted automation categories"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"automation-aug26","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"known","value":"Human-directed or supervised work represented by automation levels"},"id":"automation-ai_leads","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"ai_leads","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":378},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Person-time-weighted automation categories"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"automation-aug26","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"known","value":"Human-directed or supervised work represented by automation levels"},"id":"automation-autonomous","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"autonomous","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":378},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Estimated change in completion time"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-early25","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"time_change","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Random assignment of issues to AI allowed or disallowed"},"source_refs":[{"locator":"Methodology; Core Result","source_id":"src-productivity"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":246},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"known","value":"Chosen tools, primarily Cursor with Claude 3.5/3.7 Sonnet"},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-late25","comparability_caveats":["Selection and concurrent-agent time measurement biases."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-later-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"assessment","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Volunteer participation and self-selected submitted tasks"},"source_refs":[{"locator":"Introduction; Wider adoption section","source_id":"src-productivity-update"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"alphaevolve-2025","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"alpha-training_reduction","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"training_reduction","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"alphaevolve-2025","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"alpha-kernel_speedup","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"kernel_speedup","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Percentile rank against test-time-compute reference distribution"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-revised-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"mle-revised-max","inference_budget":{"status":"known","value":"Max reasoning effort (Figure 56); token count not reported"},"internet_access":{"status":"not_reported","value":null},"metric_id":"percentile","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Up to three leaderboard submissions"},"source_refs":[{"locator":"Section 10.1.3.5 and Figure 56, printed pages 104–105","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":72},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Human-normalised average as reported by Google"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"rebench-google-feb2026","comparability_caveats":["Task subset, per-model budgets and aggregation details are not restated. Do not join to other RE-Bench protocols."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"rebench-google-feb2026-p","inference_budget":{"status":"known","value":"Gemini 3.1 Pro evaluated in Deep Think mode; comparator’s mode not restated"},"internet_access":{"status":"not_reported","value":null},"metric_id":"human_normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed page 9, Machine Learning R&D row (Deep Think mode): human-normalised average score","source_id":"src-gemini31-card"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean Code Development rubric-node score; source reports one standard error"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-code-dev-v1","comparability_caveats":["Code Development nodes only; weak correlation with full PaperBench.","Table 6 does not specify the number of repeated runs or a Code-Dev rollout time limit. Internet access is permitted under §2.5 rules, subject to per-paper blacklists."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-code-dev-iterative","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.6, §5.3 Table 6, PaperBench Code-Dev","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Execution and Result Match rubric nodes omitted; author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Different scaffold from AIDE; three seeds versus 36 for GPT-4o/AIDE."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-4o-mlab-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"MLAB"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Different scaffold from AIDE; three seeds versus 36 for GPT-4o/AIDE."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-4o-openhands-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"OpenHands"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Approximate fitted 50% task-completion horizon"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"th-1-1-sol-2026","comparability_caveats":["The June assessment names METR’s ReAct harness family, but does not state its exact revision/configuration or task count; do not inherit January details.","Cheating treatment changes the estimate drastically; METR does not consider any reported alternatives robust."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"known","value":"Cheating attempts scored as failures"},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"th-1-1-sol-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"p50","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"METR ReAct agent harness (family; exact revision/configuration not reported)"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"50%-Time Horizon paragraph, cheating attempts scored as failures","source_id":"src-metr-gpt56-sol"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Raw cohort-specific AI-assisted task-time change"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-late25","comparability_caveats":["Severe task and participant selection effects; concurrent-agent timing may be unreliable. Not a representative current productivity effect."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-returning-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"time_change","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Volunteer participation and self-selected submitted tasks"},"source_refs":[{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"}],"split":{"status":"known","value":"Returning original-study participants"},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Raw cohort-specific AI-assisted task-time change"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-late25","comparability_caveats":["Severe task and participant selection effects; concurrent-agent timing may be unreliable. Not a representative current productivity effect."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-new-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"time_change","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Volunteer participation and self-selected submitted tasks"},"source_refs":[{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"}],"split":{"status":"known","value":"Newly recruited developers"},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean over ten tasks at medium reasoning effort"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Single configuration; not the 0.250 mean across Opus 5 effort levels."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-opus-medium-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.2, prose preceding Figure 2; highest single configuration","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean full-rubric replication score on three-paper subset; Figure 3 agent error bars are SEM over three repeats"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-human-subset-v1","comparability_caveats":["Human reference is best of three attempts after 48 tracked work hours (including unattended experiments) across a part-time four-week window; not a matched 36-hour, single-run human experiment.","Subset score must not be compared as the full 20-paper or Code-Dev result."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-human-subset-o1-iter36","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Introduction; §5.4 Human Baseline Performance; Figure 3 and caption","source_id":"src-paperbench"}],"split":{"status":"known","value":"Three-paper human comparison subset; test-time-model-adaptation excluded"},"task_count":{"status":"known","value":3},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"36 hours, extended IterativeAgent run"}}]