{ "format": "agent-article-v0", "article_id": "EA-683", "canonical_url": "/tools/ea-683-evaluation-set-aging-refresh-contamination", "page_type": "reference", "claim_ids": [], "evidence_ids": [ "EA-683-CURRENT-APPROVAL", "EA-683-CURRENT-BODY-READBACK", "EVD-EA-683-URL-84CF1132C66C", "EVD-EA-683-URL-130364A0FC6E", "EVD-EA-683-URL-9162797DA302", "EVD-EA-683-URL-5064F6E55C99", "EVD-EA-683-URL-A09FB6FD80E8", "EVD-EA-683-URL-91B5C8E2E332", "EVD-EA-683-URL-9385E26C5247" ], "unknown_ids": [], "operational_object_ids": [], "provider": [], "models": [], "plans": [], "surfaces": [], "regions": [], "current_status": "CURRENT", "observation_status": "SPEC_ONLY", "freshness": "CURRENT", "last_verified": "2026-10-03", "supersedes": [], "superseded_by": [], "provenance": [ "V22-CANDIDATE-EA683-R1-20261015-C4", "BODY-SHA256:34bea6cc47022acd4b3dc0dd66e1f48744d3ef196bab6c0e5aae2233606a1532", "SRC-EA-683-URL-84CF1132C66C", "SRC-EA-683-URL-130364A0FC6E", "SRC-EA-683-URL-9162797DA302", "SRC-EA-683-URL-5064F6E55C99", "SRC-EA-683-URL-A09FB6FD80E8", "SRC-EA-683-URL-91B5C8E2E332", "SRC-EA-683-URL-9385E26C5247" ], "article_sources": [ { "id": "EA-683-CURRENT-APPROVAL", "evidence_class": "RESEARCH", "title": "現行本文の監査・revision対応", "supports": [ "現行台帳のAUDIT_PASSと候補・revision・本文ハッシュの対応を確認しています。" ], "limit": "監査状態と来歴の記録です。外部根拠の最新性や本文の全主張を追加保証しません。", "origin_class": "AUDITED_CANONICAL_RESEARCH_PACKET", "source_or_experiment_ids": [ "V22-CANDIDATE-EA683-R1-20261015-C4" ], "supporting_locators": [ "https://drive.google.com/drive/folders/1etIlR4ks7Bu95WmR9rQV0wCVyU6UabOj", "EA-683-R1-20261003" ], "provenance_status": "CURRENT_CANONICAL_APPROVAL_READBACK" }, { "id": "EA-683-CURRENT-BODY-READBACK", "evidence_class": "IMPL", "title": "現行本文の収録照合", "supports": [ "公開生成入力の本文SHA-256が、監査済み候補に結び付いたハッシュと一致しています。" ], "limit": "本文の同一性の検査です。外部システムの挙動や全主張の真偽を実証したものではありません。", "checked_at": "2026-10-17", "origin_class": "IMPLEMENTATION_READBACK", "source_or_experiment_ids": [ "BODY-SHA256:34bea6cc47022acd4b3dc0dd66e1f48744d3ef196bab6c0e5aae2233606a1532" ], "supporting_locators": [ "content/staging/EA-683.md", "34bea6cc47022acd4b3dc0dd66e1f48744d3ef196bab6c0e5aae2233606a1532", "EA-683-R1-20261003" ], "provenance_status": "CURRENT_APPROVED_BODY_HASH_MATCH" }, { "id": "EVD-EA-683-URL-84CF1132C66C", "evidence_class": "RESEARCH", "title": "E1 | official benchmark maintainer | Epoch AI, \"FrontierMath: Tiers 1-4\" |", "href": "https://epoch.ai/frontiermath/tiers-1-4", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-84CF1132C66C" ], "supporting_locators": [ "EVIDENCE TABLE", "E1 | official benchmark maintainer | Epoch AI, \"FrontierMath: Tiers 1-4\" | https://epoch.ai/frontiermath/tiers-1-4" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-130364A0FC6E", "evidence_class": "RESEARCH", "title": "E2 | benchmark site / author-maintained | LiveBench |", "href": "https://livebench.ai/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-130364A0FC6E" ], "supporting_locators": [ "EVIDENCE TABLE", "E2 | benchmark site / author-maintained | LiveBench | https://livebench.ai/" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-9162797DA302", "evidence_class": "RESEARCH", "title": "E3 | primary paper | White et al., \"LiveBench: A Challenging, Contamination-Free LLM Benchmark\" |", "href": "https://arxiv.org/abs/2406.19314", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-9162797DA302" ], "supporting_locators": [ "EVIDENCE TABLE", "E3 | primary paper | White et al., \"LiveBench: A Challenging, Contamination-Free LLM Benchmark\" | https://arxiv.org/abs/2406.19314" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-5064F6E55C99", "evidence_class": "RESEARCH", "title": "E4 | primary paper | Yang et al., \"Rethinking Benchmark and Contamination for Language Models with Rephrased Samples\" |", "href": "https://arxiv.org/abs/2311.04850", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-5064F6E55C99" ], "supporting_locators": [ "EVIDENCE TABLE", "E4 | primary paper | Yang et al., \"Rethinking Benchmark and Contamination for Language Models with Rephrased Samples\" | https://arxiv.org/abs/2311.04850" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-A09FB6FD80E8", "evidence_class": "RESEARCH", "title": "E5 | provider evaluation audit / primary operational report | OpenAI, \"Why SWE-bench Verified no longer measures frontier coding capabilities\" |", "href": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-A09FB6FD80E8" ], "supporting_locators": [ "EVIDENCE TABLE", "E5 | provider evaluation audit / primary operational report | OpenAI, \"Why SWE-bench Verified no longer measures frontier coding capabilities\" | https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-91B5C8E2E332", "evidence_class": "RESEARCH", "title": "E6 | research paper / counterevidence | Sun et al., \"The Emperor's New Clothes in Benchmarking? A Rigorous Examination of Mitigation Strategies for LLM Benchmark Data Contamination", "href": "https://arxiv.org/abs/2503.16402", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-91B5C8E2E332" ], "supporting_locators": [ "EVIDENCE TABLE", "E6 | research paper / counterevidence | Sun et al., \"The Emperor's New Clothes in Benchmarking? A Rigorous Examination of Mitigation Strategies for LLM Benchmark Data Contamination\" | https://arxiv.org/abs/2503.16402" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-683-URL-9385E26C5247", "evidence_class": "RESEARCH", "title": "E7 | primary paper | Qian et al., \"VarBench: Robust Language Model Benchmarking Through Dynamic Variable Perturbation\" |", "href": "https://arxiv.org/abs/2406.17681", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-683-URL-9385E26C5247" ], "supporting_locators": [ "EVIDENCE TABLE", "E7 | primary paper | Qian et al., \"VarBench: Robust Language Model Benchmarking Through Dynamic Variable Perturbation\" | https://arxiv.org/abs/2406.17681" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" } ], "research_sections": [ { "heading": "要点と前提", "text": "Benchmark scores are versioned observations, not timeless model properties. Enterprise evaluation records should preserve the exact benchmark release, item-set identity or lawful hash, scorer and harness version, model/revision, tool or web policy, inference settings, run date, and environment." }, { "heading": "Why this matters", "text": "Public exposure creates contamination risk, but it does not prove that a particular model is contaminated. Benchmark maintainers can also repair defective items, change scoring, alter refresh cadence, or revise harnesses after release. Those changes can break direct longitudinal comparability even when the model is unchanged.\n\n\nFrontierMath v2 is a concrete example: its maintainer reported that the v2 release addressed errors affecting 42% of problems. LiveBench likewise uses explicit release lineage and refreshes its question set, while its original paper and current operating cadence differ. These are reasons to bind every score to the benchmark version actually used rather than silently replacing historical results." }, { "heading": "Operating rule", "text": "Treat item correction, scoring-rule change, harness/runtime change, tool or web policy change, model revision, and inference-compute change as comparison boundaries unless a bridge study demonstrates equivalence. Never overwrite an old score when a benchmark refreshes. Create a new evaluation revision and retain the old run with its original provenance.\n\n\nWhen licensing permits, a bridge study should run old and new sets under the same model, harness, and settings; report paired unchanged items separately from new or repaired items; record exclusions and reasons; and avoid collapsing non-equivalent releases into one trend line." }, { "heading": "Contamination claims", "text": "A public benchmark implies an opportunity for exposure. A model-specific contamination claim requires additional evidence such as training-data evidence, controlled canaries, memorization or overlap tests, or another specific method. Score movement after a refresh is not diagnostic: it can result from contamination removal, difficulty shift, item repair, scorer changes, or harness changes.\n\n\nFresh, private, rephrased, synthetic, and dynamically generated items can reduce some exposure or aging risks, but none is a universal contamination cure. Refresh methods can preserve leakage or change task fidelity, so their own version and generation policy belong in the run record." }, { "heading": "Failure envelope", "text": "Do not compare mixed benchmark releases without stratification. Other invalidating conditions include silent item replacement, scorer or harness changes treated as model gains or losses, benchmark saturation, leaked solutions, broken or ambiguous items, hidden tool/web-access differences, inference-compute changes, post-release benchmark edits, and non-equivalent dynamic variants.\n\n\nPrivate test sets, canaries, training-data attestations, historical harness artifacts, and item hashes may be unavailable. Record unavailable fields as UNKNOWN rather than inferring them." }, { "heading": "Safe wording", "text": "“Benchmark scores are versioned observations, not timeless model properties. Preserve benchmark-release, item/scorer/harness provenance and inference conditions. A refresh or item correction starts a new comparison revision unless equivalence is demonstrated. Public exposure is a contamination risk, not proof that a particular model is contaminated.”" }, { "heading": "Evidence basis", "text": "- Epoch AI, FrontierMath: Tiers 1–4, v2 release 2026-06-12.\n- LiveBench, release lineage observed 2026-10-03; latest release then shown as 2026-06-25.\n- White et al., “LiveBench: A Challenging, Contamination-Free LLM Benchmark,” arXiv:2406.19314.\n- Yang et al., “Rethinking Benchmark and Contamination for Language Models with Rephrased Samples,” arXiv:2311.04850.\n- OpenAI, “Why SWE-bench Verified no longer measures frontier coding capabilities,” 2026 audit report.\n- Sun et al., “The Emperor’s New Clothes in Benchmarking?,” arXiv:2503.16402.\n- Qian et al., “VarBench,” arXiv:2406.17681." }, { "heading": "Limits", "text": "This guidance does not establish the contamination status of any named closed model, does not assume access to training corpora, and does not prescribe one organization-wide refresh cadence or quantitative bridge-equivalence threshold." } ], "human_url": "/tools/ea-683-evaluation-set-aging-refresh-contamination", "markdown_url": "/agent/article.md?path=%2Ftools%2Fea-683-evaluation-set-aging-refresh-contamination", "json_url": "/agent/article.json?path=%2Ftools%2Fea-683-evaluation-set-aging-refresh-contamination" }