[{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-opus5","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-earlier","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"53.2%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":53.2},{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-mythos51","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-earlier","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"53.4%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"mythos51","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":53.4},{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-opus55","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-later","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"55.8%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus55","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":55.8},{"benchmark_version_id":"cobench-prior-unspecified","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co-prior-opus5","limitations":["Historical score restated September 22; original publication/evaluation date not established. No comparison group assigned."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co-prior","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"59.6%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":59.6},{"benchmark_version_id":"cobench-prior-unspecified","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co-prior-mythos51","limitations":["Historical score restated September 22; original publication/evaluation date not established. No comparison group assigned."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co-prior","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"57.6%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"mythos51","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":57.6},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":"debug-fig84","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"openai-research-debugging-astra","limitations":["41 debugging bugs; six additional alignment-auditing tasks are described separately."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"78.05%","source_refs":[{"locator":"§10.1.3.1; Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":78.05},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-1p-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"superseded","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.2","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"not_reported","value":null},{"benchmark_version_id":"nanogpt-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"nanogpt-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"nanogpt-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"superseded","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.3","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":null},{"benchmark_version_id":"posttrainbench-lite-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"posttrainbench-lite-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"posttrainbench-lite-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"superseded","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.4","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":null},{"benchmark_version_id":"grb-2026-08","comparison_group_id":"grb-limited","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"grb-flash37","limitations":["Bugged tasks retained; adjusted hypothetical 47% is not an observed score."],"metric_id":"pass1","operational_context":null,"outcome":"numeric","protocol_id":"grb-current","publication_date":{"precision":"month","value":"2026-08"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"27%","source_refs":[{"locator":"Printed p.37; PDF index 36; Results","source_id":"src-grb"}],"subject_id":"gemini37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":27},{"benchmark_version_id":"grb-2026-08","comparison_group_id":"grb-limited","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"grb-pro31","limitations":["Approximate preliminary estimate, not a confident baseline."],"metric_id":"pass1","operational_context":null,"outcome":"numeric","protocol_id":"grb-earlier","publication_date":{"precision":"month","value":"2026-08"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"around 16%","source_refs":[{"locator":"Printed p.37; PDF index 36; Results, footnote 10","source_id":"src-grb"}],"subject_id":"gemini31","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":16},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o3-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"2.6 ± 0.2%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.2 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":2.6},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-4o-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"4.1 ± 0.1%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-4o","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.1 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":4.1},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-gemini-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.2 ± 0.2%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-gemini","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.2 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":3.2},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"13.2 ± 0.3%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-o1","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.3 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":13.2},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-claude-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"21.0 ± 0.8%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-claude","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.8 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":21},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o3-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"8.5 ± 0.8%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o3-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.8 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":8.5},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-claude-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"16.1 ± 0.1%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-claude-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.1 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":16.1},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"24.4 ± 0.7%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o1-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.7 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":24.4},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative36-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-iter36-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative36","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"26.0 ± 0.3%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o1-iter36","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.3 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":26},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-o1-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-o1-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"16.9 ± 1.1%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-o1","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 1.1 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":16.9},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-4o-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-4o-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"8.7 ± 0.5%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-4o","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.5 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":8.7},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-llama-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-llama-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.0 ± 1.0%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-llama","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 1.0 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":3},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-claude-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-claude-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"7.6 ± 1.8%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-claude","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 1.8 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":7.6},{"benchmark_version_id":"mle-revised-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-revised-astra","limitations":["Numeric chart not transcribed; revised metric cannot be joined to medal rates."],"metric_id":"percentile","operational_context":null,"outcome":"not_reported","protocol_id":"mle-revised-p","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"superseded","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.5","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":null},{"benchmark_version_id":"rebench-gemini3","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"rebench-gemini3-assessment","limitations":["Qualitative risk assessment; exact graph scores withheld. Not comparable to full seven-task suite."],"metric_id":"assessment","operational_context":null,"outcome":"qualitative","protocol_id":"rebench-gemini-p","publication_date":{"precision":"month","value":"2025-11"},"publication_status":"published","qualitative_statement":"Google reports Gemini 3 Pro remains below its ML R&D alert threshold.","reported_value_text":null,"source_refs":[{"locator":"Machine Learning R&D, printed pp.13–15; PDF indices 13–15","source_id":"src-gemini3"}],"subject_id":"gemini3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"qualitative","value":null},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-opus45","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"289","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus45","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":110,"reported_text":"[110, 1268]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":1268},"unit":"minutes","value":289},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-opus45","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"320","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus45","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":170,"reported_text":"[170, 729]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":729},"unit":"minutes","value":320},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-gpt5","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"138","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-gpt5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":68,"reported_text":"[68, 281]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":281},"unit":"minutes","value":138},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-gpt5","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"214","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-gpt5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":117,"reported_text":"[117, 480]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":480},"unit":"minutes","value":214},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-opus4","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"86","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":44,"reported_text":"[44, 144]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":144},"unit":"minutes","value":86},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-opus4","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"101","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":58,"reported_text":"[58, 170]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":170},"unit":"minutes","value":101},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-o3","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"94","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":48,"reported_text":"[48, 165]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":165},"unit":"minutes","value":94},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-o3","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"121","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":74,"reported_text":"[74, 201]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":201},"unit":"minutes","value":121},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-sonnet37","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"56","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-sonnet37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":28,"reported_text":"[28, 94]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":94},"unit":"minutes","value":56},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-sonnet37","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"60","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-sonnet37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":32,"reported_text":"[32, 106]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":106},"unit":"minutes","value":60},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-opus-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-opus-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.250","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-opus","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.25},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-sol-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-sol-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.191","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-sol","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.191},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-kimi-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-kimi-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.174","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-kimi","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.174},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-sonnet-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-sonnet-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.145","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-sonnet","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.145},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-terra-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-terra-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.135","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-terra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.135},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-luna-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-luna-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.117","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-luna","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.117},{"benchmark_version_id":"automation-aug26","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":{"precision":"month","value":"2026-08"},"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"automation-ai_leads-aug26","limitations":["Not directly comparable across organizations without common methodology."],"metric_id":"ai_leads","operational_context":{"denominator":"Person-time-weighted categories in a fixed work basket","method":"estimate","population":"Anthropic model R&D work","sampling_method":"20% staff sampled weekly by department; Claude researches and rates tasks","timeframe":"August 2026; basket and weights constructed from July 2026"},"outcome":"numeric","protocol_id":"automation-ai_leads","publication_date":{"precision":"day","value":"2026-09-17"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"26%","source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"subject_id":"anthropic","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":26},{"benchmark_version_id":"automation-aug26","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":{"precision":"month","value":"2026-08"},"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"automation-autonomous-aug26","limitations":["Self-report using Claude judges. Zero describes fully autonomous measured work, not absence of AI assistance."],"metric_id":"autonomous","operational_context":{"denominator":"Person-time-weighted categories in a fixed work basket","method":"estimate","population":"Anthropic model R&D work","sampling_method":"20% staff sampled weekly by department; Claude researches and rates tasks","timeframe":"August 2026; basket and weights constructed from July 2026"},"outcome":"numeric","protocol_id":"automation-autonomous","publication_date":{"precision":"day","value":"2026-09-17"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"No measured subset fully autonomous","source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"subject_id":"anthropic","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":0},{"benchmark_version_id":"productivity-early25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-slowdown","limitations":["Positive means slower. Not an estimate for all developers or current tools."],"metric_id":"time_change","operational_context":{"denominator":"246 randomized real repository issues","method":"experimental","population":"16 experienced open-source developers","sampling_method":"Recruited developers; within-study issue randomization","timeframe":"Early 2025"},"outcome":"numeric","protocol_id":"productivity-p","publication_date":{"precision":"day","value":"2025-07-10"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"19% longer","source_refs":[{"locator":"Core Result","source_id":"src-productivity"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":19},{"benchmark_version_id":"productivity-late25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-selection","limitations":["Reported raw effects are not used as a comparable trend."],"metric_id":"assessment","operational_context":{"denominator":"Submitted tasks; substantial task and participant selection","method":"experimental","population":"10 returning and 47 newly recruited developers","sampling_method":"Voluntary recruitment; randomized submitted tasks","timeframe":"Experiment begun August 2025; reported February 2026"},"outcome":"qualitative","protocol_id":"productivity-later-p","publication_date":{"precision":"day","value":"2026-02-24"},"publication_status":"published","qualitative_statement":"METR reports that selection effects make the later experiment unreliable for estimating current productivity gains.","reported_value_text":null,"source_refs":[{"locator":"Introduction and selection-effects discussion","source_id":"src-productivity-update"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"qualitative","value":null},{"benchmark_version_id":"alphaevolve-2025","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"alpha-training_reduction-result","limitations":["Does not measure improvement in the optimizer across generations."],"metric_id":"training_reduction","operational_context":{"denominator":"Training time","method":"telemetry","population":"Gemini training / matrix multiplication kernel","sampling_method":"Developer-reported production application","timeframe":"Deployment reported May 2025"},"outcome":"numeric","protocol_id":"alpha-training_reduction","publication_date":{"precision":"day","value":"2025-05-14"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"1%","source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"subject_id":"alphaevolve","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":1},{"benchmark_version_id":"alphaevolve-2025","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"alpha-kernel_speedup-result","limitations":["Does not measure improvement in the optimizer across generations."],"metric_id":"kernel_speedup","operational_context":{"denominator":"Kernel performance baseline","method":"telemetry","population":"Gemini training / matrix multiplication kernel","sampling_method":"Developer-reported production application","timeframe":"Deployment reported May 2025"},"outcome":"numeric","protocol_id":"alpha-kernel_speedup","publication_date":{"precision":"day","value":"2025-05-14"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"23%","source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"subject_id":"alphaevolve","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":23},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-appendix-astra","limitations":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"66.72%","source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":"kernelgen-1p-astra","uncertainty":null,"unit":"percent","value":66.72},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-appendix-sol6","limitations":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"38.12%","source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"subject_id":"gpt6-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":38.12},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-appendix-sol56","limitations":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"61.08%","source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"subject_id":"gpt56-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":61.08},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-appendix-luna6","limitations":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"21.83%","source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"subject_id":"gpt6-luna-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":21.83},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-appendix-luna56","limitations":["Per-model output-token budgets for the appendix bars are not printed. Figure 53 plots separate budget-dependent results."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"22.4%","source_refs":[{"locator":"Appendix A.8.1.3.2, Figure 85 (PDF printed page 152); exact bar labels; section 10.1.3.2","source_id":"src-astra"}],"subject_id":"gpt56-luna-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":22.4},{"benchmark_version_id":"mle-revised-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-revised-max-astra","limitations":["Percentile rank against a generated reference distribution, not the original MLE-bench medal rate."],"metric_id":"percentile","operational_context":null,"outcome":"numeric","protocol_id":"mle-revised-max","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"93.80%","source_refs":[{"locator":"Section 10.1.3.5, Figure 56, printed page 105; GPT-6 Astra (max)","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":"mle-revised-astra","uncertainty":null,"unit":"percent","value":93.8},{"benchmark_version_id":"mle-revised-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Explicit figure label transcribed without rounding. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-revised-max-sol56","limitations":["Percentile rank against a generated reference distribution, not the original MLE-bench medal rate."],"metric_id":"percentile","operational_context":null,"outcome":"numeric","protocol_id":"mle-revised-max","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"87.18%","source_refs":[{"locator":"Section 10.1.3.5, Figure 56, printed page 105; GPT-5.6 Sol (max)","source_id":"src-astra"}],"subject_id":"gpt56-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":87.18},{"benchmark_version_id":"nanogpt-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Inspected each curve and legend in the original figure; exact score and token coordinates are not printed. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"nanogpt-curve-astra","limitations":["Checked official HTML, original PNG and report PDF; no labelled point table or downloadable structured results found. Values have not been estimated from chart heights."],"metric_id":"score","operational_context":null,"outcome":"qualitative","protocol_id":"nanogpt-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":"Results appear in a chart at several output-token budgets; exact point values are not printed.","reported_value_text":null,"source_refs":[{"locator":"Section 10.1.3.3, Figure 54; GPT-6 Astra series","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":"nanogpt-astra","uncertainty":null,"unit":null,"value":null},{"benchmark_version_id":"nanogpt-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Inspected each curve and legend in the original figure; exact score and token coordinates are not printed. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"nanogpt-curve-sol56","limitations":["Checked official HTML, original PNG and report PDF; no labelled point table or downloadable structured results found. Values have not been estimated from chart heights."],"metric_id":"score","operational_context":null,"outcome":"qualitative","protocol_id":"nanogpt-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":"Results appear in a chart at several output-token budgets; exact point values are not printed.","reported_value_text":null,"source_refs":[{"locator":"Section 10.1.3.3, Figure 54; GPT-5.6 Sol series","source_id":"src-astra"}],"subject_id":"gpt56-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":null,"value":null},{"benchmark_version_id":"posttrainbench-lite-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Inspected each curve and legend in the original figure; exact score and token coordinates are not printed. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"posttrainbench-lite-curve-astra","limitations":["Checked official HTML, original PNG and report PDF; no labelled point table or downloadable structured results found. Values have not been estimated from chart heights."],"metric_id":"score","operational_context":null,"outcome":"qualitative","protocol_id":"posttrainbench-lite-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":"Results appear in a chart at several output-token budgets; exact point values are not printed.","reported_value_text":null,"source_refs":[{"locator":"Section 10.1.3.4, Figure 55; GPT-6 Astra series","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":"posttrainbench-lite-astra","uncertainty":null,"unit":null,"value":null},{"benchmark_version_id":"posttrainbench-lite-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Inspected each curve and legend in the original figure; exact score and token coordinates are not printed. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"posttrainbench-lite-curve-sol56","limitations":["Checked official HTML, original PNG and report PDF; no labelled point table or downloadable structured results found. Values have not been estimated from chart heights."],"metric_id":"score","operational_context":null,"outcome":"qualitative","protocol_id":"posttrainbench-lite-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":"Results appear in a chart at several output-token budgets; exact point values are not printed.","reported_value_text":null,"source_refs":[{"locator":"Section 10.1.3.4, Figure 55; GPT-5.6 Sol series","source_id":"src-astra"}],"subject_id":"gpt56-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":null,"value":null},{"benchmark_version_id":"rebench-google-feb2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Exact 1.27 and 1.04 values checked against PDF page 9 and official HTML. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"gemini31-rebench-score","limitations":["Task subset, per-model budgets and aggregation details are not restated. Do not join to other RE-Bench protocols."],"metric_id":"human_normalized","operational_context":null,"outcome":"numeric","protocol_id":"rebench-google-feb2026-p","publication_date":{"precision":"day","value":"2026-02-19"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"1.27","source_refs":[{"locator":"Printed page 9, Machine Learning R&D row (Deep Think mode): human-normalised average score","source_id":"src-gemini31-card"}],"subject_id":"gemini31-rebench","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":1.27},{"benchmark_version_id":"rebench-google-feb2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Exact 1.27 and 1.04 values checked against PDF page 9 and official HTML. Second extraction check against rendered primary figure and text; not experimental replication or human review.","reviewed_at":"2026-09-27T04:15:49.829513Z","reviewer":"Codex","status":"agent_checked"},"id":"gemini3-rebench-comparator-score","limitations":["Task subset, per-model budgets and aggregation details are not restated. Do not join to other RE-Bench protocols."],"metric_id":"human_normalized","operational_context":null,"outcome":"numeric","protocol_id":"rebench-google-feb2026-p","publication_date":{"precision":"day","value":"2026-02-19"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"1.04","source_refs":[{"locator":"Printed page 9, Machine Learning R&D row (Deep Think mode): human-normalised average score","source_id":"src-gemini31-card"}],"subject_id":"gemini3-rebench-comparator","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":1.04},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":"debug-fig84","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"debug-gpt6-sol-card","limitations":["Figure labels are mean rubric rewards; do not equate with older-card median wording."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"64.20%","source_refs":[{"locator":"Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"subject_id":"gpt6-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":64.2},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":"debug-fig84","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"debug-gpt56-sol-card","limitations":["Figure labels are mean rubric rewards; do not equate with older-card median wording."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"68.32%","source_refs":[{"locator":"Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"subject_id":"gpt56-sol-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":68.32},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":"debug-fig84","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"debug-gpt6-luna-card","limitations":["Figure labels are mean rubric rewards; do not equate with older-card median wording."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"46.62%","source_refs":[{"locator":"Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"subject_id":"gpt6-luna-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":46.62},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":"debug-fig84","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"debug-gpt56-luna-card","limitations":["Figure labels are mean rubric rewards; do not equate with older-card median wording."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"50.8%","source_refs":[{"locator":"Appendix Figure 84, Internal Research Debugging, original labeled PNG","source_id":"src-astra"}],"subject_id":"gpt56-luna-card","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":50.8},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-deepseek-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"6.0 ± 0.3%","source_refs":[{"locator":"§5.2, Table 4, BasicAgent average Replication Scores","source_id":"src-paperbench"}],"subject_id":"pb-deepseek","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.3 percent","source_ref":{"locator":"§5.2, Table 4, BasicAgent average Replication Scores","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":6},{"benchmark_version_id":"paperbench-code-dev-v1","comparison_group_id":null,"empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-code-dev-result","limitations":["Code-Dev variant grades only Code Development nodes; not a full PaperBench score."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-code-dev-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"43.4 ± 0.8%","source_refs":[{"locator":"§2.6, §5.3 Table 6, PaperBench Code-Dev","source_id":"src-paperbench"}],"subject_id":"pb-o1-code-dev","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.8 percent","source_ref":{"locator":"§2.6, §5.3 Table 6, PaperBench Code-Dev","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":43.4},{"benchmark_version_id":"mle-2024","comparison_group_id":null,"empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-4o-mlab-result","limitations":["Scaffold comparison; seed count differs from AIDE."],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-4o-mlab-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.8 ± 0.5%","source_refs":[{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"}],"subject_id":"mle-4o-mlab","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 0.5 percent","source_ref":{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":0.8},{"benchmark_version_id":"mle-2024","comparison_group_id":null,"empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-4o-openhands-result","limitations":["Scaffold comparison; seed count differs from AIDE."],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-4o-openhands-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"4.4 ± 1.4%","source_refs":[{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"}],"subject_id":"mle-4o-openhands","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":null,"lower":null,"reported_text":"± 1.4 percent","source_ref":{"locator":"§3.1, Table 2, Scaffolding and Models experiments, Any Medal column","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":4.4},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-codexmax","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"173","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-codexmax","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":81,"reported_text":"[81,411]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":411},"unit":"minutes","value":173},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-sonnet45","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"122","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-sonnet45","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":59,"reported_text":"[59,252]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":252},"unit":"minutes","value":122},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-opus41","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"114","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-opus41","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":56,"reported_text":"[56,215]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":215},"unit":"minutes","value":114},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-grok4","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"109","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-grok4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":48,"reported_text":"[48,235]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":235},"unit":"minutes","value":109},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-sonnet4","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"75","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-sonnet4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":38,"reported_text":"[38,132]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":132},"unit":"minutes","value":75},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-gpt4-1106","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"8.5","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-gpt4-1106","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":4,"reported_text":"[4.0,16.1]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":16.1},"unit":"minutes","value":8.5},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-gpt4-0314","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"5.4","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-gpt4-0314","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":2.5,"reported_text":"[2.5,10.3]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":10.3},"unit":"minutes","value":5.4},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-gpt4-1106","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.6","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-gpt4-1106","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":1.6,"reported_text":"[1.6,7.5]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":7.5},"unit":"minutes","value":3.6},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-gpt4-0314","limitations":["Historical January 2026 snapshot; human-task duration, not AI runtime.","TH1 and TH1.1 task suite and infrastructure differ."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.5","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"}],"subject_id":"th-gpt4-0314","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":1.6,"reported_text":"[1.6,6.9]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates table","source_id":"src-th11"},"type":"confidence_interval","upper":6.9},"unit":"minutes","value":3.5},{"benchmark_version_id":"th-1-1-sol-2026","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-sol-gpt56","limitations":["Approximate estimate with 95% CI 5–40 hours; METR does not consider it a robust capability measurement.","Counting cheating attempts as successes gives >270 hours beyond reliable suite range; discarding them yields 71 hours (95% CI 13–11,400 hours).","June source identifies the ReAct harness family, but not its exact revision/configuration or task count."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-sol-p","publication_date":{"precision":"day","value":"2026-06-26"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"around 11.3 hours","source_refs":[{"locator":"50%-Time Horizon paragraph, cheating attempts scored as failures","source_id":"src-metr-gpt56-sol"}],"subject_id":"th-gpt56-sol","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"95%","lower":5,"reported_text":"95% CI: 5–40 hours","source_ref":{"locator":"50%-Time Horizon paragraph, cheating attempts scored as failures","source_id":"src-metr-gpt56-sol"},"type":"confidence_interval","upper":40},"unit":"hours","value":11.3},{"benchmark_version_id":"productivity-late25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-returning","limitations":["Raw estimate, not a reliable causal estimate of current developer productivity.","Study participation and submitted tasks were selected; some concurrent-agent task times are unreliable."],"metric_id":"time_change","operational_context":{"denominator":"Late-2025 participating open-source developers","method":"experimental","population":"Returning original-study participants","sampling_method":"Volunteer participants and self-selected submitted tasks","timeframe":"Late 2025"},"outcome":"numeric","protocol_id":"productivity-returning-p","publication_date":{"precision":"day","value":"2026-02-24"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"-18%","source_refs":[{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":{"level":"Source-reported confidence interval; level not specified in summary","lower":-38,"reported_text":"-38% to +9%","source_ref":{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"},"type":"confidence_interval","upper":9},"unit":"percent","value":-18},{"benchmark_version_id":"productivity-late25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-new","limitations":["Raw estimate, not a reliable causal estimate of current developer productivity.","Study participation and submitted tasks were selected; some concurrent-agent task times are unreliable."],"metric_id":"time_change","operational_context":{"denominator":"Late-2025 participating open-source developers","method":"experimental","population":"Newly recruited developers","sampling_method":"Volunteer participants and self-selected submitted tasks","timeframe":"Late 2025"},"outcome":"numeric","protocol_id":"productivity-new-p","publication_date":{"precision":"day","value":"2026-02-24"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"-4%","source_refs":[{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":{"level":"Source-reported confidence interval; level not specified in summary","lower":-15,"reported_text":"-15% to +9%","source_ref":{"locator":"Introduction, Raw results paragraph (February 24, 2026)","source_id":"src-productivity-update"},"type":"confidence_interval","upper":9},"unit":"percent","value":-4},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":null,"empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original source and exact printed value checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T04:17:28Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-opus-medium-mean","limitations":["Medium-effort configuration over 10 tasks; distinct from all-effort system mean 0.250."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-opus-medium-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.288","source_refs":[{"locator":"§3.2, prose preceding Figure 2; single best configuration","source_id":"src-ai4ai"}],"subject_id":"a4-opus-medium","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.288},{"benchmark_version_id":"paperbench-human-subset-v1","comparison_group_id":null,"empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Original PaperBench v1 introduction, §5.4, and Figure 3 checked; no human review or experimental replication claimed.","reviewed_at":"2026-09-27T05:27:25Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-human-subset-result","limitations":["Three-paper subset; 36-hour o1 IterativeAgent run, not the full 20-paper or Code-Dev result.","Human reference is best of three after 48 tracked work hours (including unattended experiments) in part-time arrangements; agent and human time accounting differs."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-human-subset-o1-iter36","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"26.6%","source_refs":[{"locator":"Introduction; §5.4 Human Baseline Performance; Figure 3 and caption","source_id":"src-paperbench"}],"subject_id":"pb-o1-human-subset","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":26.6}]