{ "format": "agent-article-v0", "article_id": "EA-111", "canonical_url": "/tools/ea-111-business-task-benchmark-library", "page_type": "reference", "claim_ids": [], "evidence_ids": [ "EVD-EA-111-URL-FC3078F3C2BA", "EVD-EA-111-URL-AC757EE4E5DE", "EA-111-R11-RESEARCH", "EA-111-R11-WEB-READBACK" ], "unknown_ids": [ "EA-111-UNKNOWN-01", "EA-111-UNKNOWN-CURRENTNESS" ], "operational_object_ids": [], "provider": [], "models": [], "plans": [], "surfaces": [], "regions": [], "current_status": "UNKNOWN", "observation_status": "SPEC_ONLY", "freshness": "UNKNOWN", "last_verified": null, "supersedes": [], "superseded_by": [], "provenance": [ "COMM-040-EA0111", "SRC-EA-111-URL-FC3078F3C2BA", "SRC-EA-111-URL-AC757EE4E5DE", "WEBBODY-EA0111-R8-8231564E" ], "article_sources": [ { "id": "EVD-EA-111-URL-FC3078F3C2BA", "evidence_class": "RESEARCH", "title": "source class: author-maintained implementation documentation; exact title: Inspect; author/org: UK AI Security Institute and Meridian Labs; canonical URL:", "href": "https://inspect.aisi.org.uk/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-111-URL-FC3078F3C2BA" ], "supporting_locators": [ "EVIDENCE TABLE", "- source class: author-maintained implementation documentation; exact title: Inspect; author/org: UK AI Security Institute and Meridian Labs; canonical URL: https://inspect.aisi.org.uk/" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EVD-EA-111-URL-AC757EE4E5DE", "evidence_class": "RESEARCH", "title": "source class: official risk-management framework page; exact title: AI Risk Management Framework; author/org: NIST; canonical URL:", "href": "https://airc.nist.gov/airmf-resources/", "supports": [], "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。", "origin_class": "SOURCE_PACKET_REFERENCE", "source_or_experiment_ids": [ "SRC-EA-111-URL-AC757EE4E5DE" ], "supporting_locators": [ "EVIDENCE TABLE", "- source class: official risk-management framework page; exact title: AI Risk Management Framework; author/org: NIST; canonical URL: https://airc.nist.gov/airmf-resources/" ], "scope": "EVIDENCE TABLE", "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED" }, { "id": "EA-111-R11-RESEARCH", "evidence_class": "RESEARCH", "title": "EA-111 監査済みresearch / canonical revision binding", "supports": [ "Inspectの一次Docsは、業務タスクをdataset（input/target等）、solver（単純生成からtoolを使うagent）、scorer（比較・model grading・custom）に分けてTaskとして再利用でき、agent evaluationではsandboxを選べることを確立する。これはbenchmarkを作る実装骨格の証拠である。一方、200+ evalsの存在は、メール・Sheets・Docs・検索/RAG・codeの各業務fixtureが同じ品質基準で揃っていること、また結果が企業業務成功率を表すことを確立しない。Inspectは評価器であって、業務要件・権限・正解データ・provider差を自動で決めるものではない。" ], "limit": "canonical監査・revision bindingの存在を示す。外部ソースcurrentnessが本文またはcontrol-planeで確定していない場合は、currentnessをUNKNOWNのまま保持する。", "origin_class": "AUDITED_CANONICAL_RESEARCH_PACKET", "source_or_experiment_ids": [ "COMM-040-EA0111" ], "supporting_locators": [ "COMM-040-EA0111", "EA-111-R5-LEGACY-MIGRATION-20261008-R1", "content/staging/EA-111.md" ], "scope": "監査済みEA本文・canonical revision・既存Source Graph relationの範囲。欠損currentnessは推測で閉じない。", "provenance_status": "AUDIT_PASS_CANONICAL_BINDING" }, { "id": "EA-111-R11-WEB-READBACK", "evidence_class": "IMPL", "title": "EA-111 Web body identity / canonical readback", "supports": [ "canonical staging bodyのSHA-256一致とR9 canonical ledger readbackをProduction projectionの入力として検証する。" ], "limit": "body identity/readbackは本文claim自体の真偽や外部ソースの最新性を追加保証しない。", "checked_at": null, "origin_class": "IMPLEMENTATION_READBACK", "source_or_experiment_ids": [ "WEBBODY-EA0111-R8-8231564E" ], "supporting_locators": [ "WEBBODY-EA0111-R8-8231564E", "content/staging/EA-111.md", "EA-111-R5-LEGACY-MIGRATION-20261008-R1" ], "scope": "R9/R10 canonical readback後のProduction生成時body identity。", "provenance_status": "CANONICAL_READBACK_BODY_HASH_MATCH" } ], "research_sections": [ { "heading": "WHY_THIS_MATTERS_TO_UPDATE", "text": "Email、Sheets、Docs、検索/RAG、codeの業務タスクを共通fixtureで比較できる棚が必要であり、製品デモや単一ベンチマークを業務品質保証へ外挿しないための評価単位を作る。" }, { "heading": "SCOPE_ROUTING", "text": "benchmark / evaluation / implementation / interoperability" }, { "heading": "CLAIM_OR_RESEARCH_QUESTION", "text": "再利用可能なbusiness-task benchmarkを、dataset・solver/agent・scorer・sandbox・logの分離で設計できるか。Inspectの提供機能は業務成功を保証するか。" }, { "heading": "EVIDENCE TABLE", "text": "- source class: author-maintained implementation documentation; exact title: Inspect; author/org: UK AI Security Institute and Meridian Labs; canonical URL: https://inspect.aisi.org.uk/; publication/update/access: current documentation crawled/accessed 2026-09-13 JST; version/release/tag/commit/API revision/model ID: page does not expose a pinned package release; current docs state retained as PRODUCT_VOLATILE; supporting locator: lines 150-164（framework、broad evaluation domains、composable datasets/agents/tools/scorers、200+ pre-built evals、sandbox） and 195-200（dataset/solver/scorer）; evidence label: OSS_PRACTICE + PRODUCT_VOLATILE; relation: support / limit.\n- source class: official risk-management framework page; exact title: AI Risk Management Framework; author/org: NIST; canonical URL: https://airc.nist.gov/airmf-resources/; publication/update/access: AI RMF 1.0 page, accessed 2026-09-13 JST; version/release/tag/commit/API revision/model ID: AI RMF 1.0; supporting locator: lines 58-63（Measure、context-specific Profiles）; evidence label: VERIFIED_SPEC + PRODUCT_VOLATILE; relation: context / limit." }, { "heading": "FINDINGS", "text": "Inspectの一次Docsは、業務タスクをdataset（input/target等）、solver（単純生成からtoolを使うagent）、scorer（比較・model grading・custom）に分けてTaskとして再利用でき、agent evaluationではsandboxを選べることを確立する。これはbenchmarkを作る実装骨格の証拠である。一方、200+ evalsの存在は、メール・Sheets・Docs・検索/RAG・codeの各業務fixtureが同じ品質基準で揃っていること、また結果が企業業務成功率を表すことを確立しない。Inspectは評価器であって、業務要件・権限・正解データ・provider差を自動で決めるものではない。" }, { "heading": "LIMITS / COUNTEREVIDENCE / FAILURE ENVELOPE", "text": "model-graded scorerはjudge bias/評価漏れを含み得る。業務の正解が複数ある場合、文字一致だけでは不十分。sandbox内の成功は本番権限・外部API・tenant dataへの安全性を証明しない。Docsの「200+」は更新で変わるため固定値にしない。" }, { "heading": "AVAILABILITY_GATES", "text": "公開DocsとOSS frameworkは無認証で閲覧可。実行には環境、モデルprovider/API key、依存package、sandbox runtimeが必要。企業データ・外部connector・認証済みWorkspaceへの実行は本Packetの範囲外。" }, { "heading": "IMPLEMENTATION_STATE", "text": "業務fixture schema（入力、前提、許可tool、期待状態、禁止副作用、証拠ログ、scorer、human review）を設計する段階。Inspect上での実行は未実施。" }, { "heading": "BENCHMARK_OR_EVAL", "text": "一業務一fixtureで、①タスク達成、②禁止副作用、③引用/根拠、④tool sequence、⑤人間承認、⑥再試行時の復旧を別scoreにする。pass/failだけでなくunknownとabstainを保存し、model leaderboardへ変換しない。実測なし。" }, { "heading": "SECURITY / PRIVACY / AUDIT / COST / OPERABILITY", "text": "SECURITY/PRIVACY: synthetic fixtureとsandboxを先行。AUDIT: dataset/solver/scorer/package/model/config/run logを固定。COST: 試行回数、tool call、judge callを計上。OPERABILITY: version lock、seed/config保存、failure artifact保全、再実行可能なCLIを必要条件とする。" }, { "heading": "SAFE_WORDING_FOR_FUTURE_BOOK", "text": "「Inspectはdataset・solver・scorer・agent/tool・sandboxを組み合わせる評価実装の選択肢である。企業の業務品質を示すbenchmark libraryは、業務fixture、権限境界、禁止副作用、採点基準、human reviewを組織側で定義して初めて成立し、Inspectの既成eval数だけでは業務保証にならない。」" }, { "heading": "UPDATE_WORK_HANDOFF", "text": "次回はEA-111の業務カテゴリごとに、最小synthetic fixtureを一件ずつ作り、同一model/configと複数model/providerで同じscorerを走らせるTEST_PROFILEを追加する。実データ接続は後続の承認済みWorkへ分離する。" }, { "heading": "OPEN_UNKNOWN", "text": "企業業務fixtureの横断標準、RAG/検索のground-truth設計、judge agreement、各providerのtool trace完全性、Inspect現行releaseと互換性。" }, { "heading": "STOP_CONDITION", "text": "pre-built evalの件数や単一runのscoreを、企業業務全体・全provider・本番安全性へ外挿しない。\nUNIT_CLOSE_STATUS: INDIVIDUAL_BOUNDED_CLOSE / EVIDENCE_RECORDED / UNKNOWN_PRESERVED\n\n---" }, { "heading": "AUDITED_CURRENTNESS_BOUNDARY", "text": "Inspect-style evaluation framework capabilities/pre-built eval counts are PRODUCT_VOLATILE and do not define enterprise business truth." } ], "human_url": "/tools/ea-111-business-task-benchmark-library", "markdown_url": "/agent/article.md?path=%2Ftools%2Fea-111-business-task-benchmark-library", "json_url": "/agent/article.json?path=%2Ftools%2Fea-111-business-task-benchmark-library" }