{ "format": "agent-article-v0", "article_id": "EA-687", "canonical_url": "/tools/ea-687-ai-test-mutation-testing", "page_type": "reference", "claim_ids": [], "evidence_ids": [ "EVD-EA-687-URL-BF7291989124", "EVD-EA-687-URL-157D6FB1C022", "EVD-EA-687-URL-2CAA81CB4B32", "EVD-EA-687-URL-5A6A9FA3860C", "EA-687-R11-RESEARCH", "EA-687-R11-WEB-READBACK" ], "unknown_ids": [ "EA-687-UNKNOWN-01" ], "operational_object_ids": [], "provider": [], "models": [], "plans": [], "surfaces": [], "regions": [], "current_status": "CURRENT", "observation_status": "SPEC_ONLY", "freshness": "CURRENT", "last_verified": "2026-10-04", "supersedes": [], "superseded_by": [], "provenance": [ "EA-687-FIRST-WORK-20261004", "SRC-EA-687-URL-BF7291989124", "SRC-EA-687-URL-157D6FB1C022", "SRC-EA-687-URL-2CAA81CB4B32", "SRC-EA-687-URL-5A6A9FA3860C", "WEBBODY-EA0687-R8-299B9ADE" ], "article_sources": [ { "id": "EVD-EA-687-URL-BF7291989124", "evidence_class": "RESEARCH", "title": "S1: Foster et al., Mutation-Guided LLM-based Test Generation at Meta, v1, §5.3/§6/Table8 —", "href": "https://arxiv.org/html/2501.12862v1", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-687-URL-BF7291989124" ], "supporting_locators": [ "SOURCES", "S1", "- S1: Foster et al., Mutation-Guided LLM-based Test Generation at Meta, v1, §5.3/§6/Table8 — https://arxiv.org/html/2501.12862v1" ], "scope": "SOURCES", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-687-URL-157D6FB1C022", "evidence_class": "RESEARCH", "title": "S2: Hamidi et al., How effective are traditional test criteria at detecting bugs in large language models generated code?, v1, RQ3 —", "href": "https://arxiv.org/html/2609.09315v1", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-687-URL-157D6FB1C022" ], "supporting_locators": [ "SOURCES", "S2", "- S2: Hamidi et al., How effective are traditional test criteria at detecting bugs in large language models generated code?, v1, RQ3 — https://arxiv.org/html/2609.09315v1" ], "scope": "SOURCES", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-687-URL-2CAA81CB4B32", "evidence_class": "RESEARCH", "title": "S3: Stryker Mutator / Mutant states and metrics —", "href": "https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-687-URL-2CAA81CB4B32" ], "supporting_locators": [ "SOURCES", "S3", "- S3: Stryker Mutator / Mutant states and metrics — https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/" ], "scope": "SOURCES", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-687-URL-5A6A9FA3860C", "evidence_class": "RESEARCH", "title": "S4: Alshahwan et al., TestGen-LLM industrial report（要旨範囲）—", "href": "https://arxiv.org/abs/2402.09171", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-687-URL-5A6A9FA3860C" ], "supporting_locators": [ "SOURCES", "S4", "- S4: Alshahwan et al., TestGen-LLM industrial report（要旨範囲）— https://arxiv.org/abs/2402.09171" ], "scope": "SOURCES", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EA-687-R11-RESEARCH", "evidence_class": "RESEARCH", "title": "EA-687 監査済みresearch / canonical revision binding", "supports": [ "Meta ACHの産業研究は、line coverageを増やさずmutantを殺すtestにも価値があると報告しています。equivalent mutant判定は対象と前処理に依存し、一般的な等価性判定能力の証明ではありません。[S1]" ], "limit": "canonical監査・revision bindingの存在を示す。外部ソースcurrentnessが本文またはcontrol-planeで確定していない場合は、currentnessをUNKNOWNのまま保持する。", "checked_at": "2026-10-04", "origin_class": "AUDITED_CANONICAL_RESEARCH_PACKET", "source_or_experiment_ids": [ "EA-687-FIRST-WORK-20261004" ], "supporting_locators": [ "EA-687-FIRST-WORK-20261004", "content/staging/EA-687.md" ], "scope": "監査済みEA本文・canonical revision・既存Source Graph relationの範囲。欠損currentnessは推測で閉じない。", "provenance_status": "AUDIT_PASS_CANONICAL_BINDING" }, { "id": "EA-687-R11-WEB-READBACK", "evidence_class": "IMPL", "title": "EA-687 Web body identity / canonical readback", "supports": [ "canonical staging bodyのSHA-256一致とR9 canonical ledger readbackをProduction projectionの入力として検証する。" ], "limit": "body identity/readbackは本文claim自体の真偽や外部ソースの最新性を追加保証しない。", "checked_at": "2026-10-04", "origin_class": "IMPLEMENTATION_READBACK", "source_or_experiment_ids": [ "WEBBODY-EA0687-R8-299B9ADE" ], "supporting_locators": [ "WEBBODY-EA0687-R8-299B9ADE", "content/staging/EA-687.md", "EA-687-FIRST-WORK-20261004" ], "scope": "R9/R10 canonical readback後のProduction生成時body identity。", "provenance_status": "CANONICAL_READBACK_BODY_HASH_MATCH" } ], "research_sections": [ { "heading": "PUBLIC_CLAIM", "text": "Mutation Testingは選んだ変異をtestが捉えるかを測ります。高いscoreだけでは要求適合や現実の不具合検出を保証しません。AI生成testのassertionは、実装自身の出力から独立した要求・期待結果と照合します。" }, { "heading": "ESTABLISHED", "text": "Meta ACHの産業研究は、line coverageを増やさずmutantを殺すtestにも価値があると報告しています。equivalent mutant判定は対象と前処理に依存し、一般的な等価性判定能力の証明ではありません。[S1]\n\n2026 preprintはcode/test双方LLM生成のbenchmark設定で、faultを実行することとoracleが誤りを検出することを分離し、oracle failureを報告しています。Metaの成熟codebaseでのconcern-focused実験と同じ母集団ではありません。[S2]\n\nStryker docsではdetected=killed+timeout、valid=detected+survived+no coverage、score=detected/validです。runtime/compile error、ignored/pending、covered-only scoreを別に記録します。[S3]" }, { "heading": "INFERENCE", "text": "要求に接地したoracle、baseline安定性、mutant集合・operator・状態・分母、人間review、実bug regressionを別gateとして運用します。compile PASS、test件数、coverage増分、mutation scoreの一つだけで採用を決めません。" }, { "heading": "FAILURE_ENVELOPE", "text": "誤ったbaselineを正解とするassertion、easy mutant偏重、invalid/ignored除外、equivalent mutant、flakiness、timeoutとインフラ障害の混同が判断を歪めます。surviveだけでtest不足と断言せず、等価性や仕様範囲も確認します。" }, { "heading": "IMPLEMENTATION_STATE", "text": "資料照合とsynthetic反例のみです。10%割引要求に違反する20%割引baselineを誤oracleが通し、選んだ2mutantを全て殺しました（選択集合score100）。独立要求oracleはそのbaselineを拒否し正しい実装を通しました。実LLM、Stryker、ACH、real-bug corpusやproductionでの検出率は測定していません。" }, { "heading": "AVAILABILITY_GATES", "text": "tool/release/operator、baseline SHA、要求locator、oracle review、mutant diff、timeout分類、raw result、eligible denominator、実bug regression、コードの外部送信許可、runtime/推論/review費を確認します。" }, { "heading": "UNKNOWN", "text": "対象製品のreal-bug detectionと費用、現行全LLM、2026 preprintの査読・raw再現、Stryker CLI release、実test flakiness、全repo regressionは未確認です。" }, { "heading": "STOP_CONDITION", "text": "高coverage・高score・人間acceptanceを「バグなし」「要求適合」「security保証」へ変換しないでください。staging candidateは未監査・未公開です。" }, { "heading": "SOURCES", "text": "- S1: Foster et al., Mutation-Guided LLM-based Test Generation at Meta, v1, §5.3/§6/Table8 — https://arxiv.org/html/2501.12862v1\n- S2: Hamidi et al., How effective are traditional test criteria at detecting bugs in large language models generated code?, v1, RQ3 — https://arxiv.org/html/2609.09315v1\n- S3: Stryker Mutator / Mutant states and metrics — https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/\n- S4: Alshahwan et al., TestGen-LLM industrial report（要旨範囲）— https://arxiv.org/abs/2402.09171\n- T1: EA-687_Evidence_Packet.md / fixtures/mutation_oracle.py / results/EA-687.json（監査bundle内）" } ], "human_url": "/tools/ea-687-ai-test-mutation-testing", "markdown_url": "/agent/article.md?path=%2Ftools%2Fea-687-ai-test-mutation-testing", "json_url": "/agent/article.json?path=%2Ftools%2Fea-687-ai-test-mutation-testing" }