{
 "or_jev_114": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.635,
   "macro_f1": 0.6304,
   "nll": 0.8229,
   "brier": 0.488,
   "ece": 0.0543,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6744,
   "acc_at_50cov": 0.76
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.935,
   "nll": 0.2022,
   "brier": 0.1013,
   "ece": 0.0465,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9282,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1643
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.885,
   "macro_f1": 0.8885,
   "nll": 0.2847,
   "brier": 0.1522,
   "ece": 0.0498,
   "p_true_zero": 0.0,
   "conf_auroc": 0.935,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.9349,
   "nll": 0.2007,
   "brier": 0.1098,
   "ece": 0.0601,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8904,
   "acc_at_50cov": 1.0
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.865,
   "macro_f1": 0.865,
   "nll": 0.3147,
   "brier": 0.1911,
   "ece": 0.0584,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8387,
   "acc_at_50cov": 0.98
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.76,
   "macro_f1": 0.8816,
   "nll": 0.5538,
   "brier": 0.3201,
   "ece": 0.0319,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8216,
   "acc_at_50cov": 0.955
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.9,
   "macro_f1": 0.9,
   "nll": 0.2577,
   "brier": 0.1504,
   "ece": 0.0593,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8653,
   "acc_at_50cov": 0.995
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.935,
   "nll": 0.2022,
   "brier": 0.1013,
   "ece": 0.0465,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9282,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1643
  },
  "suite:img_refcoco": {
   "n": 200,
   "failed": 0,
   "acc": 0.785,
   "mean_iou": 0.7026,
   "ece": 0.3034,
   "mean_conf": 0.4816,
   "conf_auroc": 0.804
  },
  "type:box": {
   "n": 200,
   "failed": 0,
   "acc": 0.785,
   "mean_iou": 0.7026,
   "ece": 0.3034,
   "mean_conf": 0.4816,
   "conf_auroc": 0.804
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.851,
   "macro_f1": 0.8983,
   "nll": 0.365,
   "brier": 0.2085,
   "ece": 0.0287,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8686,
   "acc_at_50cov": 0.988,
   "score_mae": 0.1643
  },
  "cost": {
   "usd": 0.0556,
   "input_tokens": 1323095,
   "output_tokens": 103054,
   "calls": 1200
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 182.6,
   "p95": 306.5
  }
 },
 "or_jev": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.59,
   "macro_f1": 0.5733,
   "nll": 0.8845,
   "brier": 0.5153,
   "ece": 0.1054,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7342,
   "acc_at_50cov": 0.75
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.9432,
   "nll": 0.2124,
   "brier": 0.1069,
   "ece": 0.046,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8677,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1633
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.835,
   "macro_f1": 0.8468,
   "nll": 0.4128,
   "brier": 0.2019,
   "ece": 0.0507,
   "p_true_zero": 0.005,
   "conf_auroc": 0.9143,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.945,
   "nll": 0.1768,
   "brier": 0.0952,
   "ece": 0.0596,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8877,
   "acc_at_50cov": 1.0
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8799,
   "nll": 0.2936,
   "brier": 0.1817,
   "ece": 0.0297,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8237,
   "acc_at_50cov": 0.98
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.7125,
   "macro_f1": 0.8431,
   "nll": 0.6486,
   "brier": 0.3586,
   "ece": 0.069,
   "p_true_zero": 0.0025,
   "conf_auroc": 0.8389,
   "acc_at_50cov": 0.92
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.9125,
   "macro_f1": 0.9125,
   "nll": 0.2352,
   "brier": 0.1385,
   "ece": 0.0447,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8539,
   "acc_at_50cov": 0.985
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.9432,
   "nll": 0.2124,
   "brier": 0.1069,
   "ece": 0.046,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8677,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1633
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.839,
   "macro_f1": 0.8582,
   "nll": 0.396,
   "brier": 0.2202,
   "ece": 0.0283,
   "p_true_zero": 0.001,
   "conf_auroc": 0.8711,
   "acc_at_50cov": 0.988,
   "score_mae": 0.1633
  },
  "cost": {
   "usd": 0.0464,
   "input_tokens": 1105273,
   "output_tokens": 99065,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 192.2,
   "p95": 326.3
  }
 },
 "luna": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.67,
   "macro_f1": 0.6656,
   "nll": 0.8231,
   "brier": 0.4816,
   "ece": 0.1351,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6523,
   "acc_at_50cov": 0.76
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.85,
   "macro_f1": 0.8513,
   "nll": 0.4542,
   "brier": 0.2198,
   "ece": 0.0902,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9006,
   "acc_at_50cov": 1.0,
   "score_mae": 0.3219
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.87,
   "macro_f1": 0.8681,
   "nll": 0.4636,
   "brier": 0.189,
   "ece": 0.0424,
   "p_true_zero": 0.01,
   "conf_auroc": 0.8426,
   "acc_at_50cov": 0.97
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 1,
   "acc": 0.885,
   "macro_f1": 0.8843,
   "nll": 0.5988,
   "brier": 0.1682,
   "ece": 0.0553,
   "p_true_zero": 0.03,
   "conf_auroc": 0.7334,
   "acc_at_50cov": 0.94
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 1,
   "acc": 0.805,
   "macro_f1": 0.8048,
   "nll": 0.5003,
   "brier": 0.2852,
   "ece": 0.0823,
   "p_true_zero": 0.005,
   "conf_auroc": 0.7666,
   "acc_at_50cov": 0.92
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.77,
   "macro_f1": 0.8595,
   "nll": 0.6434,
   "brier": 0.3353,
   "ece": 0.0795,
   "p_true_zero": 0.005,
   "conf_auroc": 0.7743,
   "acc_at_50cov": 0.915
  },
  "type:noul": {
   "n": 400,
   "failed": 2,
   "acc": 0.845,
   "macro_f1": 0.8445,
   "nll": 0.5496,
   "brier": 0.2267,
   "ece": 0.0553,
   "p_true_zero": 0.0175,
   "conf_auroc": 0.7683,
   "acc_at_50cov": 0.935
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.85,
   "macro_f1": 0.8513,
   "nll": 0.4542,
   "brier": 0.2198,
   "ece": 0.0902,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9006,
   "acc_at_50cov": 1.0,
   "score_mae": 0.3219
  },
  "all": {
   "n": 1000,
   "failed": 2,
   "acc": 0.816,
   "macro_f1": 0.8644,
   "nll": 0.568,
   "brier": 0.2688,
   "ece": 0.053,
   "p_true_zero": 0.009,
   "conf_auroc": 0.7869,
   "acc_at_50cov": 0.946,
   "score_mae": 0.3219
  },
  "cost": {
   "usd": 0.0471,
   "input_tokens": 471262,
   "output_tokens": 0,
   "calls": 998
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 220.7,
   "p95": 582.6
  }
 },
 "pplx_v11": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.51,
   "macro_f1": 0.5065,
   "nll": 1.0073,
   "brier": 0.6287,
   "ece": 0.176,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5914,
   "acc_at_50cov": 0.56
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.942,
   "nll": 0.1692,
   "brier": 0.1023,
   "ece": 0.0484,
   "p_true_zero": 0.0,
   "conf_auroc": 0.899,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1122
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.84,
   "macro_f1": 0.8347,
   "nll": 0.4402,
   "brier": 0.2279,
   "ece": 0.0879,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8759,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.96,
   "macro_f1": 0.96,
   "nll": 0.1381,
   "brier": 0.0758,
   "ece": 0.0407,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8717,
   "acc_at_50cov": 0.99
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.895,
   "macro_f1": 0.895,
   "nll": 0.2597,
   "brier": 0.1669,
   "ece": 0.0384,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8436,
   "acc_at_50cov": 1.0
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.675,
   "macro_f1": 0.8237,
   "nll": 0.7238,
   "brier": 0.4283,
   "ece": 0.0866,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7462,
   "acc_at_50cov": 0.82
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.9275,
   "macro_f1": 0.9275,
   "nll": 0.1989,
   "brier": 0.1214,
   "ece": 0.0394,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8651,
   "acc_at_50cov": 0.995
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.942,
   "nll": 0.1692,
   "brier": 0.1023,
   "ece": 0.0484,
   "p_true_zero": 0.0,
   "conf_auroc": 0.899,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1122
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.83,
   "macro_f1": 0.8498,
   "nll": 0.4029,
   "brier": 0.2403,
   "ece": 0.0561,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8538,
   "acc_at_50cov": 0.992,
   "score_mae": 0.1122
  },
  "cost": {
   "usd": 0.0086,
   "input_tokens": 432475,
   "output_tokens": 1000,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 302.0,
   "p95": 464.7
  }
 },
 "clef": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.35,
   "macro_f1": 0.28,
   "nll": 1.438,
   "brier": 0.8358,
   "ece": 0.3598,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6344,
   "acc_at_50cov": 0.42
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.9,
   "macro_f1": 0.8934,
   "nll": 0.3759,
   "brier": 0.1638,
   "ece": 0.1332,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9228,
   "acc_at_50cov": 1.0,
   "score_mae": 0.3328
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.825,
   "macro_f1": 0.8128,
   "nll": 0.5934,
   "brier": 0.258,
   "ece": 0.1479,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8934,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.955,
   "macro_f1": 0.955,
   "nll": 0.1549,
   "brier": 0.0802,
   "ece": 0.0388,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8554,
   "acc_at_50cov": 1.0
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.875,
   "macro_f1": 0.875,
   "nll": 0.3132,
   "brier": 0.1932,
   "ece": 0.0247,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8071,
   "acc_at_50cov": 0.99
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.5875,
   "macro_f1": 0.7892,
   "nll": 1.0157,
   "brier": 0.5469,
   "ece": 0.1205,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6792,
   "acc_at_50cov": 0.715
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.915,
   "macro_f1": 0.915,
   "nll": 0.234,
   "brier": 0.1367,
   "ece": 0.0114,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8281,
   "acc_at_50cov": 0.995
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.9,
   "macro_f1": 0.8934,
   "nll": 0.3759,
   "brier": 0.1638,
   "ece": 0.1332,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9228,
   "acc_at_50cov": 1.0,
   "score_mae": 0.3328
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.781,
   "macro_f1": 0.8238,
   "nll": 0.5751,
   "brier": 0.3062,
   "ece": 0.0358,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8071,
   "acc_at_50cov": 0.944,
   "score_mae": 0.3328
  },
  "cost": {
   "usd": 0.1378,
   "input_tokens": 574275,
   "output_tokens": 0,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 502.1,
   "p95": 1089.1
  }
 },
 "clef_flash": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.335,
   "macro_f1": 0.2548,
   "nll": 1.4591,
   "brier": 0.9112,
   "ece": 0.4527,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6943,
   "acc_at_50cov": 0.44
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8795,
   "nll": 0.6402,
   "brier": 0.2784,
   "ece": 0.2926,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9196,
   "acc_at_50cov": 1.0,
   "score_mae": 0.6699
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.85,
   "macro_f1": 0.8394,
   "nll": 0.7048,
   "brier": 0.2734,
   "ece": 0.2317,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8684,
   "acc_at_50cov": 0.99
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.955,
   "macro_f1": 0.955,
   "nll": 0.1753,
   "brier": 0.0893,
   "ece": 0.0505,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7807,
   "acc_at_50cov": 0.98
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8797,
   "nll": 0.3098,
   "brier": 0.1901,
   "ece": 0.0192,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8011,
   "acc_at_50cov": 0.98
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.5925,
   "macro_f1": 0.8166,
   "nll": 1.0819,
   "brier": 0.5923,
   "ece": 0.1961,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5068,
   "acc_at_50cov": 0.59
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.9175,
   "macro_f1": 0.9174,
   "nll": 0.2425,
   "brier": 0.1397,
   "ece": 0.0305,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8105,
   "acc_at_50cov": 0.98
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8795,
   "nll": 0.6402,
   "brier": 0.2784,
   "ece": 0.2926,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9196,
   "acc_at_50cov": 1.0,
   "score_mae": 0.6699
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.78,
   "macro_f1": 0.8342,
   "nll": 0.6578,
   "brier": 0.3485,
   "ece": 0.094,
   "p_true_zero": 0.0,
   "conf_auroc": 0.68,
   "acc_at_50cov": 0.852,
   "score_mae": 0.6699
  },
  "cost": {
   "usd": 0.0218,
   "input_tokens": 574275,
   "output_tokens": 0,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 429.4,
   "p95": 720.4
  }
 },
 "clef_omni": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.285,
   "macro_f1": 0.1479,
   "nll": 2.3968,
   "brier": 1.2612,
   "ece": 0.6355,
   "p_true_zero": 0.0,
   "conf_auroc": 0.565,
   "acc_at_50cov": 0.34
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.68,
   "macro_f1": 0.6767,
   "nll": 0.9259,
   "brier": 0.4315,
   "ece": 0.0473,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8248,
   "acc_at_50cov": 0.89,
   "score_mae": 0.7656
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8569,
   "nll": 0.8016,
   "brier": 0.3102,
   "ece": 0.3575,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8828,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.895,
   "macro_f1": 0.8944,
   "nll": 0.335,
   "brier": 0.1879,
   "ece": 0.0547,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7115,
   "acc_at_50cov": 0.95
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.87,
   "macro_f1": 0.8699,
   "nll": 0.33,
   "brier": 0.1979,
   "ece": 0.0543,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8133,
   "acc_at_50cov": 0.96
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.5825,
   "macro_f1": 0.8173,
   "nll": 1.5992,
   "brier": 0.7857,
   "ece": 0.4926,
   "p_true_zero": 0.0,
   "conf_auroc": 0.2493,
   "acc_at_50cov": 0.285
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.8825,
   "macro_f1": 0.8822,
   "nll": 0.3325,
   "brier": 0.1929,
   "ece": 0.0434,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7655,
   "acc_at_50cov": 0.955
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.68,
   "macro_f1": 0.6767,
   "nll": 0.9259,
   "brier": 0.4315,
   "ece": 0.0473,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8248,
   "acc_at_50cov": 0.89,
   "score_mae": 0.7656
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.722,
   "macro_f1": 0.8262,
   "nll": 0.9579,
   "brier": 0.4777,
   "ece": 0.2059,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5929,
   "acc_at_50cov": 0.736,
   "score_mae": 0.7656
  },
  "cost": {
   "usd": 0.0859,
   "input_tokens": 572877,
   "output_tokens": 0,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 274.3,
   "p95": 399.5
  }
 },
 "mercury": {
  "suite:img_ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.39,
   "macro_f1": 0.3565,
   "nll": 1.3359,
   "brier": 0.8227,
   "ece": 0.3747,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6665,
   "acc_at_50cov": 0.47
  },
  "suite:img_count": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.9329,
   "nll": 0.1813,
   "brier": 0.0954,
   "ece": 0.0141,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9358,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1059
  },
  "suite:img_pets": {
   "n": 200,
   "failed": 0,
   "acc": 0.845,
   "macro_f1": 0.8286,
   "nll": 0.4243,
   "brier": 0.2219,
   "ece": 0.0574,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8876,
   "acc_at_50cov": 1.0
  },
  "suite:img_sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.93,
   "macro_f1": 0.93,
   "nll": 0.1827,
   "brier": 0.0952,
   "ece": 0.0284,
   "p_true_zero": 0.0,
   "conf_auroc": 0.861,
   "acc_at_50cov": 0.98
  },
  "suite:img_vqa_yesno": {
   "n": 200,
   "failed": 0,
   "acc": 0.87,
   "macro_f1": 0.87,
   "nll": 0.3459,
   "brier": 0.2006,
   "ece": 0.0791,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8568,
   "acc_at_50cov": 1.0
  },
  "type:choice": {
   "n": 400,
   "failed": 0,
   "acc": 0.6175,
   "macro_f1": 0.8102,
   "nll": 0.8801,
   "brier": 0.5223,
   "ece": 0.2062,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8099,
   "acc_at_50cov": 0.855
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.9,
   "macro_f1": 0.9,
   "nll": 0.2643,
   "brier": 0.1479,
   "ece": 0.0489,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8594,
   "acc_at_50cov": 0.99
  },
  "type:score": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.9329,
   "nll": 0.1813,
   "brier": 0.0954,
   "ece": 0.0141,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9358,
   "acc_at_50cov": 1.0,
   "score_mae": 0.1059
  },
  "all": {
   "n": 1000,
   "failed": 0,
   "acc": 0.794,
   "macro_f1": 0.8352,
   "nll": 0.494,
   "brier": 0.2872,
   "ece": 0.0987,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8805,
   "acc_at_50cov": 0.994,
   "score_mae": 0.1059
  },
  "cost": {
   "usd": 0,
   "input_tokens": 483228,
   "output_tokens": 1000,
   "calls": 1000
  },
  "data_sha256": "9fcf4def08b3c6b1f2a043897d65c6721f4d146c2b3eff18214a1f7fa6ad992e",
  "latency_ms": {
   "p50": 305.3,
   "p95": 689.2
  }
 }
}