{
  "schema_version": 1,
  "suite": "bench-v4",
  "model_id": "jev-1.13",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "overall": {
    "rows": 1071,
    "correct": 990,
    "accuracy": 0.9243697478991597,
    "wilson95": [
      0.9069732206842946,
      0.9387327884616112
    ],
    "errors": 0,
    "macro_category_accuracy": 0.8890771967012168,
    "macro_task_f1": 0.9177322114114683
  },
  "by_category": [
    {
      "id": "agents",
      "name": "AI agents & evals",
      "rows": 154,
      "correct": 135,
      "accuracy": 0.8766233766233766,
      "wilson95": [
        0.8153385729602641,
        0.9195754167610767
      ]
    },
    {
      "id": "commerce",
      "name": "Sales & commerce",
      "rows": 90,
      "correct": 84,
      "accuracy": 0.9333333333333333,
      "wilson95": [
        0.8620973805849326,
        0.9690904934496107
      ]
    },
    {
      "id": "data",
      "name": "Data & analytics",
      "rows": 126,
      "correct": 124,
      "accuracy": 0.9841269841269841,
      "wilson95": [
        0.9439701231255323,
        0.9956362896112178
      ]
    },
    {
      "id": "design",
      "name": "Design",
      "rows": 30,
      "correct": 14,
      "accuracy": 0.4666666666666667,
      "wilson95": [
        0.30232127224128236,
        0.6385798671847496
      ]
    },
    {
      "id": "documents",
      "name": "Documents & meetings",
      "rows": 87,
      "correct": 76,
      "accuracy": 0.8735632183908046,
      "wilson95": [
        0.7876199981340796,
        0.9279112122364997
      ]
    },
    {
      "id": "engineering",
      "name": "Engineering",
      "rows": 150,
      "correct": 146,
      "accuracy": 0.9733333333333334,
      "wilson95": [
        0.9334450938998244,
        0.9895822277088953
      ]
    },
    {
      "id": "finance",
      "name": "Finance",
      "rows": 121,
      "correct": 120,
      "accuracy": 0.9917355371900827,
      "wilson95": [
        0.9546682408538117,
        0.9985396642034766
      ]
    },
    {
      "id": "legal",
      "name": "Legal",
      "rows": 126,
      "correct": 121,
      "accuracy": 0.9603174603174603,
      "wilson95": [
        0.9104631226702407,
        0.9829331386204399
      ]
    },
    {
      "id": "product",
      "name": "Product management",
      "rows": 60,
      "correct": 47,
      "accuracy": 0.7833333333333333,
      "wilson95": [
        0.6637973165265748,
        0.8687707895359299
      ]
    },
    {
      "id": "safety",
      "name": "Trust & safety",
      "rows": 60,
      "correct": 58,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8863601247107462,
        0.9908108735098498
      ]
    },
    {
      "id": "support",
      "name": "Customer support",
      "rows": 67,
      "correct": 65,
      "accuracy": 0.9701492537313433,
      "wilson95": [
        0.8975324995811698,
        0.991775477652546
      ]
    }
  ],
  "by_task": [
    {
      "id": "AGT-1",
      "name": "Which tool to call",
      "category": "agents",
      "rows": 36,
      "correct": 35,
      "accuracy": 0.9722222222222222,
      "wilson95": [
        0.8582996626496918,
        0.9950797197044302
      ]
    },
    {
      "id": "AGT-2",
      "name": "Did the injection succeed?",
      "category": "agents",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "AGT-3",
      "name": "Did the agent complete the task?",
      "category": "agents",
      "rows": 30,
      "correct": 23,
      "accuracy": 0.7666666666666667,
      "wilson95": [
        0.5907132766967574,
        0.8820776078949878
      ]
    },
    {
      "id": "AGT-4",
      "name": "Is the answer backed by sources?",
      "category": "agents",
      "rows": 30,
      "correct": 21,
      "accuracy": 0.7,
      "wilson95": [
        0.5212387898833981,
        0.8333543735604105
      ]
    },
    {
      "id": "AGT-5",
      "name": "Which reply is better?",
      "category": "agents",
      "rows": 28,
      "correct": 26,
      "accuracy": 0.9285714285714286,
      "wilson95": [
        0.7735426157408214,
        0.9801883525522292
      ]
    },
    {
      "id": "COM-1",
      "name": "Does this product match the query?",
      "category": "commerce",
      "rows": 30,
      "correct": 24,
      "accuracy": 0.8,
      "wilson95": [
        0.6269395261897256,
        0.9049502189759876
      ]
    },
    {
      "id": "COM-2",
      "name": "What type of product is this?",
      "category": "commerce",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "COM-3",
      "name": "Which industry is this company in?",
      "category": "commerce",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "DAT-1",
      "name": "Which SQL answers the question?",
      "category": "data",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "DAT-2",
      "name": "Which cell answers the question?",
      "category": "data",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "DAT-3",
      "name": "Does the chart support the claim?",
      "category": "data",
      "rows": 32,
      "correct": 32,
      "accuracy": 1.0,
      "wilson95": [
        0.8928172849426366,
        1
      ]
    },
    {
      "id": "DAT-4",
      "name": "What does this column hold?",
      "category": "data",
      "rows": 34,
      "correct": 33,
      "accuracy": 0.9705882352941176,
      "wilson95": [
        0.8508409778365519,
        0.9947892280744034
      ]
    },
    {
      "id": "DOC-1",
      "name": "Where is this page from?",
      "category": "documents",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "DOC-2",
      "name": "Decision, action item, or neither?",
      "category": "documents",
      "rows": 30,
      "correct": 21,
      "accuracy": 0.7,
      "wilson95": [
        0.5212387898833981,
        0.8333543735604105
      ]
    },
    {
      "id": "DOC-3",
      "name": "Which summary is this meeting's?",
      "category": "documents",
      "rows": 27,
      "correct": 25,
      "accuracy": 0.9259259259259259,
      "wilson95": [
        0.7663002270926937,
        0.9794457783026165
      ]
    },
    {
      "id": "DSN-1",
      "name": "What does this icon mean?",
      "category": "design",
      "rows": 30,
      "correct": 14,
      "accuracy": 0.4666666666666667,
      "wilson95": [
        0.30232127224128236,
        0.6385798671847496
      ]
    },
    {
      "id": "ENG-1",
      "name": "Which file must change?",
      "category": "engineering",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "ENG-2",
      "name": "What kind of change is this?",
      "category": "engineering",
      "rows": 30,
      "correct": 28,
      "accuracy": 0.9333333333333333,
      "wilson95": [
        0.7867618227892791,
        0.9815233646723066
      ]
    },
    {
      "id": "ENG-3",
      "name": "Which weakness class is this?",
      "category": "engineering",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "ENG-4",
      "name": "Live secret or placeholder?",
      "category": "engineering",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "ENG-5",
      "name": "Which message describes this diff?",
      "category": "engineering",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "FIN-1",
      "name": "Which part of the 10-K?",
      "category": "finance",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "FIN-2",
      "name": "What does this 8-K report?",
      "category": "finance",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "FIN-3",
      "name": "Which figure is the total?",
      "category": "finance",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "FIN-4",
      "name": "What does this number report?",
      "category": "finance",
      "rows": 31,
      "correct": 31,
      "accuracy": 1.0,
      "wilson95": [
        0.889740999265246,
        0.9999999999999999
      ]
    },
    {
      "id": "LEG-1",
      "name": "Does the NDA say this?",
      "category": "legal",
      "rows": 33,
      "correct": 28,
      "accuracy": 0.8484848484848485,
      "wilson95": [
        0.6907978686153953,
        0.9334964074203904
      ]
    },
    {
      "id": "LEG-2",
      "name": "What kind of clause is this?",
      "category": "legal",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "LEG-3",
      "name": "Is this term unfair, and how?",
      "category": "legal",
      "rows": 33,
      "correct": 33,
      "accuracy": 1.0,
      "wilson95": [
        0.8957265699643882,
        1
      ]
    },
    {
      "id": "LEG-4",
      "name": "Does the contract cover this?",
      "category": "legal",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "PRD-1",
      "name": "Which headline won?",
      "category": "product",
      "rows": 30,
      "correct": 17,
      "accuracy": 0.5666666666666667,
      "wilson95": [
        0.3919700954540873,
        0.7262276256938489
      ]
    },
    {
      "id": "PRD-2",
      "name": "Major, minor or patch?",
      "category": "product",
      "rows": 30,
      "correct": 30,
      "accuracy": 1.0,
      "wilson95": [
        0.8864829086095221,
        1
      ]
    },
    {
      "id": "SAF-1",
      "name": "Attack on the assistant",
      "category": "safety",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "SAF-2",
      "name": "Toxic comment or not",
      "category": "safety",
      "rows": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667,
      "wilson95": [
        0.8332924860304112,
        0.9940915620051425
      ]
    },
    {
      "id": "SUP-1",
      "name": "Which team owns this complaint?",
      "category": "support",
      "rows": 35,
      "correct": 35,
      "accuracy": 1.0,
      "wilson95": [
        0.9010957324106112,
        1
      ]
    },
    {
      "id": "SUP-2",
      "name": "What is the complaint about?",
      "category": "support",
      "rows": 32,
      "correct": 30,
      "accuracy": 0.9375,
      "wilson95": [
        0.7985253275199642,
        0.9826897968048427
      ]
    }
  ],
  "by_source_kind": [
    {
      "id": "real",
      "name": "real",
      "rows": 1071,
      "correct": 990,
      "accuracy": 0.9243697478991597,
      "wilson95": [
        0.9069732206842946,
        0.9387327884616112
      ]
    }
  ],
  "calibration": {
    "brier": 0.1119507328231432,
    "log_loss": 0.2162607225565288,
    "ece": 0.013102453102453044,
    "high_confidence_errors": 15
  },
  "cost": {
    "usd": 0.046292988,
    "coverage": 1.0,
    "basis": [
      "price_table_estimate"
    ],
    "per_1000_rows_usd": 0.04322407843137255
  },
  "latency_ms": {
    "p50": 438.5520829819143,
    "p95": 545.4284790030215,
    "p99": 733.5082454665087,
    "mean": 452.9427299646654
  },
  "tokens": {
    "input": 1102214,
    "output": 63528,
    "cached_input": null,
    "reasoning_output": null,
    "coverage": 1.0
  },
  "attempts": {
    "attempts": 1071,
    "retries": 0,
    "attempt_errors": 0,
    "incomplete_attempts": 0
  }
}