{
  "format": "agent-article-v0",
  "article_id": "EA-124",
  "canonical_url": "/tools/ea-124-judge-critic",
  "page_type": "reference",
  "claim_ids": [],
  "evidence_ids": [
    "EVD-EA-124-URL-16E11E26BE5B",
    "EA-124-R11-RESEARCH",
    "EA-124-R11-WEB-READBACK"
  ],
  "unknown_ids": [
    "EA-124-UNKNOWN-01",
    "EA-124-UNKNOWN-CURRENTNESS"
  ],
  "operational_object_ids": [],
  "provider": [],
  "models": [],
  "plans": [],
  "surfaces": [],
  "regions": [],
  "current_status": "UNKNOWN",
  "observation_status": "SPEC_ONLY",
  "freshness": "UNKNOWN",
  "last_verified": null,
  "supersedes": [],
  "superseded_by": [],
  "provenance": [
    "COMM-042-EA0124",
    "SRC-EA-124-URL-16E11E26BE5B",
    "WEBBODY-EA0124-R8-8D4BE9D7"
  ],
  "article_sources": [
    {
      "id": "EVD-EA-124-URL-16E11E26BE5B",
      "evidence_class": "RESEARCH",
      "title": "source class: primary research paper; exact title: Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena; author/org: Lianmin Zheng et al.; canonical URL:",
      "href": "https://arxiv.org/abs/2306.05685",
      "supports": [],
      "limit": "元の監査済み資料に記載された参照先です。URLの復元は、資料の最新性や全主張の支持を新たに検証したことを意味しません。",
      "origin_class": "SOURCE_PACKET_REFERENCE",
      "source_or_experiment_ids": [
        "SRC-EA-124-URL-16E11E26BE5B"
      ],
      "supporting_locators": [
        "EVIDENCE TABLE",
        "- source class: primary research paper; exact title: Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena; author/org: Lianmin Zheng et al.; canonical URL: https://arxiv.org/abs/2306.05685"
      ],
      "scope": "EVIDENCE TABLE",
      "provenance_status": "SOURCE_PACKET_REFERENCE_PROJECTED"
    },
    {
      "id": "EA-124-R11-RESEARCH",
      "evidence_class": "RESEARCH",
      "title": "EA-124 監査済みresearch / canonical revision binding",
      "supports": [
        "原著は、open-ended assistantを評価するための強いLLM judgeの利用を検討し、position・verbosity・self-enhancement等のbiasとlimited reasoningを明示する。同時に、特定条件ではGPT-4 judgeがhuman preferenceと80%超で一致したと報告する。したがってjudge/critic分離はscalable evaluationの候補だが、judge分離だけで独立性・妥当性・人間評価代替が成立するわけではない。generatorとjudgeの同一family、prompt、position、rubric、referenceの影響を記録する必要がある。"
      ],
      "limit": "canonical監査・revision bindingの存在を示す。外部ソースcurrentnessが本文またはcontrol-planeで確定していない場合は、currentnessをUNKNOWNのまま保持する。",
      "origin_class": "AUDITED_CANONICAL_RESEARCH_PACKET",
      "source_or_experiment_ids": [
        "COMM-042-EA0124"
      ],
      "supporting_locators": [
        "COMM-042-EA0124",
        "EA-124-R5-LEGACY-MIGRATION-20261008-R1",
        "content/staging/EA-124.md"
      ],
      "scope": "監査済みEA本文・canonical revision・既存Source Graph relationの範囲。欠損currentnessは推測で閉じない。",
      "provenance_status": "AUDIT_PASS_CANONICAL_BINDING"
    },
    {
      "id": "EA-124-R11-WEB-READBACK",
      "evidence_class": "IMPL",
      "title": "EA-124 Web body identity / canonical readback",
      "supports": [
        "canonical staging bodyのSHA-256一致とR9 canonical ledger readbackをProduction projectionの入力として検証する。"
      ],
      "limit": "body identity/readbackは本文claim自体の真偽や外部ソースの最新性を追加保証しない。",
      "checked_at": null,
      "origin_class": "IMPLEMENTATION_READBACK",
      "source_or_experiment_ids": [
        "WEBBODY-EA0124-R8-8D4BE9D7"
      ],
      "supporting_locators": [
        "WEBBODY-EA0124-R8-8D4BE9D7",
        "content/staging/EA-124.md",
        "EA-124-R5-LEGACY-MIGRATION-20261008-R1"
      ],
      "scope": "R9/R10 canonical readback後のProduction生成時body identity。",
      "provenance_status": "CANONICAL_READBACK_BODY_HASH_MATCH"
    }
  ],
  "research_sections": [
    {
      "heading": "WHY_THIS_MATTERS_TO_UPDATE",
      "text": "generatorとjudge/criticを分けると評価独立性を設計しやすいが、judge自身の位置・冗長性・self-enhancement・推論限界を管理しないと評価が見かけだけになる。"
    },
    {
      "heading": "SCOPE_ROUTING",
      "text": "multi-LLM / evaluation / governance / implementation / audit"
    },
    {
      "heading": "CLAIM_OR_RESEARCH_QUESTION",
      "text": "LLM-as-a-judge原著が確認したjudgeの有用性とbiasは何か。generatorとcriticを分ければ人間評価を代替できるか。"
    },
    {
      "heading": "EVIDENCE TABLE",
      "text": "- source class: primary research paper; exact title: Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena; author/org: Lianmin Zheng et al.; canonical URL: https://arxiv.org/abs/2306.05685; publication/update/access: submitted 2023-06-09, revised v4 2023-12-24 / accessed 2026-09-13 JST; version/release/tag/commit/API revision/model ID: arXiv v4 / NeurIPS 2023 Datasets and Benchmarks Track; supporting locator: abstract lines 8-23（position/verbosity/self-enhancement bias、limited reasoning、MT-Bench/Chatbot Arena、GPT-4 judgeとhuman preferenceの80%超 agreementの報告）; evidence label: RESEARCH; relation: support / counter / limit."
    },
    {
      "heading": "FINDINGS",
      "text": "原著は、open-ended assistantを評価するための強いLLM judgeの利用を検討し、position・verbosity・self-enhancement等のbiasとlimited reasoningを明示する。同時に、特定条件ではGPT-4 judgeがhuman preferenceと80%超で一致したと報告する。したがってjudge/critic分離はscalable evaluationの候補だが、judge分離だけで独立性・妥当性・人間評価代替が成立するわけではない。generatorとjudgeの同一family、prompt、position、rubric、referenceの影響を記録する必要がある。"
    },
    {
      "heading": "LIMITS / COUNTEREVIDENCE / FAILURE ENVELOPE",
      "text": "agreementはhuman preference近似であり、客観的正解・安全性・法的適合ではない。judge bias mitigationが必要で、論文のbenchmark/task/model条件以外へ外挿不可。criticがgeneratorと同じ誤りを持つ、rubricを読めない、危険な副作用を見落とす可能性は残る。"
    },
    {
      "heading": "AVAILABILITY_GATES",
      "text": "judge model/API/version、rubric、reference answer、conflict-of-interest、cost、privacy、human adjudication、業務データの外部送信を確認する。実judge run未実施。"
    },
    {
      "heading": "IMPLEMENTATION_STATE",
      "text": "generatorとjudgeのmodel/role分離、rubric version、pairwise/pointwise、blind position、multi-judge disagreement、human escalationを設計候補とする。"
    },
    {
      "heading": "BENCHMARK_OR_EVAL",
      "text": "objective correctness、citation entailment、policy violation、tool side effect、human preference、judge-human agreement、inter-rater reliability、bias slicesを別scoreにする。judge score単独で採否しない。"
    },
    {
      "heading": "SECURITY / PRIVACY / AUDIT / COST / OPERABILITY",
      "text": "SECURITY/PRIVACY: judgeへ送る最小redacted artifact、no secrets。AUDIT: generator/judge IDs、prompts/rubrics、positions、raw scores、human adjudication。COST: judge calls、multi-judge、human tie-break。OPERABILITY: disagreement threshold、fallback manual review、rubric migration."
    },
    {
      "heading": "SAFE_WORDING_FOR_FUTURE_BOOK",
      "text": "「LLM-as-a-judge研究は、特定benchmarkで人間選好との高い一致を報告する一方、position・verbosity・self-enhancement biasと推論限界も明示する。generator/critic分離は評価設計の一案であり、客観的正解・安全性・法的適合の自動保証ではない。」"
    },
    {
      "heading": "UPDATE_WORK_HANDOFF",
      "text": "本文改訂では judge対象、rubric、reference、blind化、bias、human tie-break、agreementを明記し、単一judgeのscoreを採否の唯一根拠にしない。"
    },
    {
      "heading": "OPEN_UNKNOWN",
      "text": "domain-specific objective scoring、judge model drift、prompt/rubric sensitivity、multi-judge calibration、safety/side-effect detection、privacy-preserving judge。"
    },
    {
      "heading": "STOP_CONDITION",
      "text": "judge score・human preference一致・無alertを、正しさ・安全・コンプライアンスの保証へ変換しない。\nUNIT_CLOSE_STATUS: INDIVIDUAL_BOUNDED_CLOSE / EVIDENCE_RECORDED / UNKNOWN_PRESERVED\n\n---"
    }
  ],
  "human_url": "/tools/ea-124-judge-critic",
  "markdown_url": "/agent/article.md?path=%2Ftools%2Fea-124-judge-critic",
  "json_url": "/agent/article.json?path=%2Ftools%2Fea-124-judge-critic"
}