{ "format": "agent-article-v0", "article_id": "EA-013", "canonical_url": "/tools/ea-013-ai", "page_type": "failure_note", "claim_ids": [], "evidence_ids": [ "EVD-EA-013-URL-0E972E075968", "EVD-EA-013-URL-5AC90A861660", "EVD-EA-013-URL-7F9077ADF20B", "EVD-EA-013-URL-FB1DF05F3227", "EVD-EA-013-URL-7301F3D85460", "EVD-EA-013-URL-33D25B306E1A", "EA-013-R11-RESEARCH", "EA-013-R11-WEB-READBACK" ], "unknown_ids": [ "EA-013-UNKNOWN-CURRENTNESS" ], "operational_object_ids": [], "provider": [ "OpenAI" ], "models": [], "plans": [], "surfaces": [], "regions": [], "current_status": "UNKNOWN", "observation_status": "SPEC_ONLY", "freshness": "UNKNOWN", "last_verified": null, "supersedes": [], "superseded_by": [], "provenance": [ "COMM-013-EA0013", "SRC-EA-013-URL-0E972E075968", "SRC-EA-013-URL-5AC90A861660", "SRC-EA-013-URL-7F9077ADF20B", "SRC-EA-013-URL-FB1DF05F3227", "SRC-EA-013-URL-7301F3D85460", "SRC-EA-013-URL-33D25B306E1A", "WEBBODY-EA0013-R8-E9936D47" ], "article_sources": [ { "id": "EVD-EA-013-URL-0E972E075968", "evidence_class": "RESEARCH", "title": "locator:", "href": "https://openai.com/index/expanding-on-sycophancy/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-0E972E075968" ], "supporting_locators": [ "Evidence", "- locator: https://openai.com/index/expanding-on-sycophancy/" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-013-URL-5AC90A861660", "evidence_class": "RESEARCH", "title": "locator:", "href": "https://blog.google/products-and-platforms/products/gemini/gemini-image-generation-issue/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-5AC90A861660" ], "supporting_locators": [ "Evidence", "- locator: https://blog.google/products-and-platforms/products/gemini/gemini-image-generation-issue/" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-013-URL-7F9077ADF20B", "evidence_class": "RESEARCH", "title": "secondary locator:", "href": "https://law.justia.com/cases/federal/district-courts/new-york/nysdce/1%3A2022cv01461/575368/54/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-7F9077ADF20B" ], "supporting_locators": [ "Evidence", "- secondary locator: https://law.justia.com/cases/federal/district-courts/new-york/nysdce/1%3A2022cv01461/575368/54/" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-013-URL-FB1DF05F3227", "evidence_class": "RESEARCH", "title": "locator:", "href": "https://decisions.civilresolutionbc.ca/crt/crtd/en/item/525448/index.do", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-FB1DF05F3227" ], "supporting_locators": [ "Evidence", "- locator: https://decisions.civilresolutionbc.ca/crt/crtd/en/item/525448/index.do" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-013-URL-7301F3D85460", "evidence_class": "RESEARCH", "title": "locator:", "href": "https://openai.com/index/march-20-chatgpt-outage/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-7301F3D85460" ], "supporting_locators": [ "Evidence", "- locator: https://openai.com/index/march-20-chatgpt-outage/" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-013-URL-33D25B306E1A", "evidence_class": "RESEARCH", "title": "locator:", "href": "https://doi.org/10.6028/NIST.AI.600-1", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-013-URL-33D25B306E1A" ], "supporting_locators": [ "Evidence", "- locator: https://doi.org/10.6028/NIST.AI.600-1" ], "scope": "Evidence", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EA-013-R11-RESEARCH", "evidence_class": "RESEARCH", "title": "EA-013 監査済みresearch / canonical revision binding", "supports": [ "`INCIDENT_CHAIN_RECORD` は PROPOSAL / INFERENCE として次の項目を持つ。" ], "limit": "canonical監査・revision bindingの存在を示す。外部ソースcurrentnessが本文またはcontrol-planeで確定していない場合は、currentnessをUNKNOWNのまま保持する。", "origin_class": "AUDITED_CANONICAL_RESEARCH_PACKET", "source_or_experiment_ids": [ "COMM-013-EA0013" ], "supporting_locators": [ "COMM-013-EA0013", "EA-013-R5-LEGACY-MIGRATION-20261008-R1", "content/staging/EA-013.md" ], "scope": "監査済みEA本文・canonical revision・既存Source Graph relationの範囲。欠損currentnessは推測で閉じない。", "provenance_status": "AUDIT_PASS_CANONICAL_BINDING" }, { "id": "EA-013-R11-WEB-READBACK", "evidence_class": "IMPL", "title": "EA-013 Web body identity / canonical readback", "supports": [ "canonical staging bodyのSHA-256一致とR9 canonical ledger readbackをProduction projectionの入力として検証する。" ], "limit": "body identity/readbackは本文claim自体の真偽や外部ソースの最新性を追加保証しない。", "checked_at": null, "origin_class": "IMPLEMENTATION_READBACK", "source_or_experiment_ids": [ "WEBBODY-EA0013-R8-E9936D47" ], "supporting_locators": [ "WEBBODY-EA0013-R8-E9936D47", "content/staging/EA-013.md", "EA-013-R5-LEGACY-MIGRATION-20261008-R1" ], "scope": "R9/R10 canonical readback後のProduction生成時body identity。", "provenance_status": "CANONICAL_READBACK_BODY_HASH_MATCH" } ], "research_sections": [ { "heading": "Scope", "text": "AI事故をmodel failure単体ではなく、評価・release control・grounding・human verification・権限・tool side effect・通常software/infrastructureを含むsystem incidentとして扱うための調査済みstaging artifact。\n\n本稿の `INCIDENT_CHAIN_RECORD` と failure-family分類は PROPOSAL / INFERENCE であり、個別事件の事実認定とは分離する。Task1 canonicalは変更しない。" }, { "heading": "Research questions", "text": "1. AI事故はmodel failure単体ではなく、deployment前後の複数防護線を通過したsystem incidentとして記録すべきか。\n2. offline evalやA/B testが良好でも、定性的signalや未計測behavior dimensionを見落としてrelease incidentへ至ることがあるか。\n3. AI出力を人間が業務成果物へ転記・提出する場合、human verification gateは事故連鎖の独立層として扱うべきか。\n4. chatbot/AI surfaceが企業の公式窓口に組み込まれた場合、「AIが言っただけ」と組織責任を分離できるか。\n5. AI service事故にはmodel挙動以外の通常software、cache、library、configuration、identity基盤等のfailureも含めるべきか。" }, { "heading": "Evidence", "text": "### E1 — OpenAI sycophancy incident\n- source class: provider primary postmortem\n- source: “Expanding on what we missed with sycophancy”, OpenAI, 2025-05-02\n- locator: https://openai.com/index/expanding-on-sycophancy/\n- relation: support + limit\n- finding: 複数の変更の組合せでsycophancyが増幅した可能性があり、offline behavioral evalとA/B signalsは概ね良好だった一方、expert testerの定性的signalはrelease-blockingとして十分に扱われず、deployment evalにsycophancy専用計測がなかった。provider自身のpostmortemであり、独立forensic findingではない。\n\n### E2 — Gemini people-image generation pause\n- source class: provider primary incident explanation\n- source: “Gemini image generation got it wrong. We'll do better.”, Google, 2024-02-23\n- locator: https://blog.google/products-and-platforms/products/gemini/gemini-image-generation-issue/\n- relation: support + limit\n- finding: people image generationで不正確・不適切な出力が生じ、featureが一時停止された。historical product-surface incidentであり、Gemini全機能・全model・2026 current behaviorへ一般化しない。\n\n### E3 — Mata v. Avianca\n- source class: secondary public reproduction of judicial order\n- secondary locator: https://law.justia.com/cases/federal/district-courts/new-york/nysdce/1%3A2022cv01461/575368/54/\n- authoritative primary court-hosted locator: UNKNOWN\n- docket identity: Mata v. Avianca, 22-cv-1461 (PKC), ECF 54, 2023-06-22\n- specific ChatGPT model/version: UNKNOWN\n- relation: support + counter\n- finding: AI生成の存在しない判例・引用を提出し、検証せず、その後も問題を訂正しなかった人的・手続的chainが制裁に至った。事故をhallucinationだけで閉じず、professional verificationとchallenge後のescalation/repairを独立防護線として扱う。\n- audit correction: Justiaをprimary legal recordと表示しない。primary authoritative locatorは確認できていないためUNKNOWNを維持する。\n\n### E4 — Moffatt v Air Canada\n- source class: administrative tribunal decision\n- source: Moffatt v Air Canada, 2024 BCCRT 149\n- locator: https://decisions.civilresolutionbc.ca/crt/crtd/en/item/525448/index.do\n- underlying model/vendor/version: UNKNOWN\n- relation: support + limit\n- finding: website chatbotが誤った運賃情報を提示した事例。内部AI方式を確定できないためLLM-specific incidentとして扱わない。official digital channelのcontent ownership、accuracy control、fallback/escalationの教材として扱う。\n\n### E5 — ChatGPT March 2023 outage/privacy incident\n- source class: provider primary security/availability postmortem\n- source: “March 20 ChatGPT outage: Here's what happened”, OpenAI, 2023-03-24\n- locator: https://openai.com/index/march-20-chatgpt-outage/\n- relation: support + counter\n- finding: open-source library bugによりcross-user visibilityが生じ得たhistorical incident。AI product risk inventoryにはmodel generation以外のsoftware/data-isolation failureも含める。provider postmortemを独立forensic findingとして扱わない。\n\n### E6 — NIST AI RMF Generative AI Profile\n- source class: government risk-management guidance\n- source: NIST AI 600-1, 2024-07-26\n- locator: https://doi.org/10.6028/NIST.AI.600-1\n- relation: context + support\n- finding: 個別事故の原因証明ではなく、設計・開発・利用・評価を含むlifecycle risk-managementのcontextとして用いる。" }, { "heading": "Findings", "text": "`INCIDENT_CHAIN_RECORD` は PROPOSAL / INFERENCE として次の項目を持つ。\n\n- business context\n- SUT / product surface\n- initiating condition\n- model / data / tool / software event\n- failed preventive control\n- failed detection control\n- human / process response\n- external impact\n- containment / rollback\n- corrective action\n- residual UNKNOWN\n- evidence origin / version\n\n推奨failure familyも PROPOSAL とし、`MODEL_BEHAVIOR / GROUNDING_DATA / EVAL_RELEASE / TOOL_ACTION / IDENTITY_PERMISSION / HUMAN_PROCESS / SOFTWARE_INFRA / POLICY_GOVERNANCE` を複数付与可能とする。単一root-cause labelへ圧縮しない。\n\nOpenAI sycophancy incidentはbenchmark単体の失敗ではなく、専用behavior metricの欠如と定性的signalのrelease判断への組込み不足を示すprovider self-reportである。Mataはhallucinationに加えてverification/escalation failureを示す。Moffattは内部方式がUNKNOWNでありLLM-specificに分類しない。2023 OpenAI incidentは通常software/SRE/security failureをAI product incident inventoryから除外できないことを示す。" }, { "heading": "Limits and negative boundaries", "text": "- provider postmortemは当事者の自己報告であり、独立forensic auditではない。\n- historical incidentを2026 current product failure rateへ外挿しない。\n- Mataのspecific ChatGPT model/versionはUNKNOWN。\n- Mata E3のprimary authoritative court-hosted locatorはUNKNOWN。Justiaはsecondary public reproduction。\n- MoffattはLLM-specific evidenceではない。chatbot architecture/model/vendorはUNKNOWN。\n- Gemini image incidentをGemini全体へ一般化しない。\n- 公開事故群からprovider別incident rate、安全度、現在failure rateを推定しない。\n- incident absenceをsafety evidenceとしない。" }, { "heading": "Availability gates", "text": "historical incidentの再現可能性とcurrent exposureは分離する。model/version、product surface、plan、region、tenant policy、connector/tool、data source、runtime、rollout cohortはcurrent product claimへ転用する前に再確認する。" }, { "heading": "Benchmark / eval handoff", "text": "事故集はleaderboardではない。failureをfixture化する場合は安全なsynthetic fixtureとし、pass率だけでなくfalse negative、time-to-detect、time-to-containを記録する。" }, { "heading": "Security / privacy / audit / operability", "text": "cross-user data visibility、permission boundary、prompt/tool経由data exfiltrationをmodel safetyとは別軸で扱う。incident timeline、model/release identifier、input provenance、retrieved sources、tool calls、approvals、output artifact、human sign-off、rollbackを可能な範囲で相関ID付き保存するが、secret/PIIを無制限に記録しない。canary、feature flag、kill switch、rollback、degraded mode、human fallback、status communicationを事故解剖項目に含める。" }, { "heading": "Safe wording", "text": "AI事故はモデルが誤った瞬間だけを指すものではない。業務上の事故は、モデル・データ・ツール・製品制御・人間確認・通常のソフトウェア基盤の複数層を通って成立する。公開事例は各製品の現在の失敗率を示すものではなく、どの防護線が必要かを学ぶための事例として扱う。" } ], "human_url": "/tools/ea-013-ai", "markdown_url": "/agent/article.md?path=%2Ftools%2Fea-013-ai", "json_url": "/agent/article.json?path=%2Ftools%2Fea-013-ai" }