{
 "laya_router": {
  "suite:ag_news": {
   "n": 200,
   "failed": 0,
   "acc": 0.925,
   "macro_f1": 0.9235,
   "nll": 0.2614,
   "brier": 0.1251,
   "ece": 0.0276,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8483,
   "acc_at_50cov": 0.98
  },
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 0,
   "acc": 0.325,
   "macro_f1": 0.3216,
   "nll": 3.4895,
   "brier": 1.0218,
   "ece": 0.4236,
   "p_true_zero": 0.035,
   "conf_auroc": 0.7779,
   "acc_at_50cov": 0.48
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.39,
   "macro_f1": 0.3734,
   "nll": 3.3999,
   "brier": 0.9813,
   "ece": 0.4059,
   "p_true_zero": 0.04,
   "conf_auroc": 0.7332,
   "acc_at_50cov": 0.57
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.82,
   "macro_f1": 0.8197,
   "nll": 0.4631,
   "brier": 0.276,
   "ece": 0.0504,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6923,
   "acc_at_50cov": 0.9
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 0,
   "acc": 0.315,
   "macro_f1": 0.3086,
   "nll": 2.0953,
   "brier": 0.8773,
   "ece": 0.1893,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5922,
   "acc_at_50cov": 0.35
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.825,
   "macro_f1": 0.6311,
   "nll": 0.4697,
   "brier": 0.2711,
   "ece": 0.0828,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7404,
   "acc_at_50cov": 0.92
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.37,
   "macro_f1": 0.3247,
   "nll": 1.8519,
   "brier": 0.9834,
   "ece": 0.4557,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5279,
   "acc_at_50cov": 0.37
  },
  "suite:banking77": {
   "n": 200,
   "failed": 0,
   "acc": 0.44,
   "macro_f1": 0.3583,
   "nll": 5.7434,
   "brier": 1.0258,
   "ece": 0.4897,
   "p_true_zero": 0.315,
   "conf_auroc": 0.7366,
   "acc_at_50cov": 0.61
  },
  "suite:boolq": {
   "n": 200,
   "failed": 0,
   "acc": 0.815,
   "macro_f1": 0.7999,
   "nll": 0.495,
   "brier": 0.2833,
   "ece": 0.0931,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7427,
   "acc_at_50cov": 0.91
  },
  "suite:emotion": {
   "n": 200,
   "failed": 0,
   "acc": 0.535,
   "macro_f1": 0.4096,
   "nll": 2.0299,
   "brier": 0.7745,
   "ece": 0.351,
   "p_true_zero": 0.01,
   "conf_auroc": 0.6832,
   "acc_at_50cov": 0.68
  },
  "suite:sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.885,
   "macro_f1": 0.8848,
   "nll": 0.3272,
   "brier": 0.1796,
   "ece": 0.0491,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7686,
   "acc_at_50cov": 0.95
  },
  "suite:sst5": {
   "n": 200,
   "failed": 0,
   "acc": 0.34,
   "macro_f1": 0.2142,
   "nll": 2.445,
   "brier": 0.9805,
   "ece": 0.381,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6236,
   "acc_at_50cov": 0.42,
   "score_mae": 1.248
  },
  "suite:typed-decisions": {
   "n": 2000,
   "failed": 0,
   "acc": 0.3635,
   "macro_f1": 0.3273,
   "nll": 1.3232,
   "brier": 0.7489,
   "ece": 0.1727,
   "p_true_zero": 0.0005,
   "conf_auroc": 0.5824,
   "acc_at_50cov": 0.433,
   "score_mae": 0.9158,
   "kl_gold": 0.5688,
   "soft_acc": 0.3487
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.68,
   "macro_f1": 0.6693,
   "nll": 1.1275,
   "brier": 0.5112,
   "ece": 0.2341,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8127,
   "acc_at_50cov": 0.9067
  },
  "suite:xnli_en": {
   "n": 150,
   "failed": 0,
   "acc": 0.8733,
   "macro_f1": 0.8708,
   "nll": 0.3339,
   "brier": 0.1837,
   "ece": 0.043,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8616,
   "acc_at_50cov": 0.9733
  },
  "suite:xnli_fr": {
   "n": 150,
   "failed": 0,
   "acc": 0.76,
   "macro_f1": 0.7513,
   "nll": 0.8519,
   "brier": 0.3876,
   "ece": 0.1667,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8029,
   "acc_at_50cov": 0.9333
  },
  "type:choice": {
   "n": 2450,
   "failed": 0,
   "acc": 0.482,
   "macro_f1": 0.3852,
   "nll": 2.0654,
   "brier": 0.7369,
   "ece": 0.251,
   "p_true_zero": 0.0327,
   "conf_auroc": 0.732,
   "acc_at_50cov": 0.6629,
   "kl_gold": 0.6845,
   "soft_acc": 0.2723
  },
  "type:noul": {
   "n": 1400,
   "failed": 0,
   "acc": 0.6871,
   "macro_f1": 0.6871,
   "nll": 0.6157,
   "brier": 0.4042,
   "ece": 0.091,
   "p_true_zero": 0.0007,
   "conf_auroc": 0.7047,
   "acc_at_50cov": 0.8286,
   "kl_gold": 0.4259,
   "soft_acc": 0.4661
  },
  "type:score": {
   "n": 1000,
   "failed": 0,
   "acc": 0.328,
   "macro_f1": 0.242,
   "nll": 1.6855,
   "brier": 0.8448,
   "ece": 0.2296,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5415,
   "acc_at_50cov": 0.364,
   "score_mae": 0.9823,
   "kl_gold": 0.5892,
   "soft_acc": 0.3181
  },
  "workflow:agent_trace_observability": {
   "n": 500,
   "failed": 0,
   "acc": 0.394,
   "macro_f1": 0.3342,
   "nll": 1.3359,
   "brier": 0.7454,
   "ece": 0.1926,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5683,
   "acc_at_50cov": 0.456,
   "score_mae": 0.8585,
   "kl_gold": 0.4715,
   "soft_acc": 0.3739
  },
  "workflow:customer_service": {
   "n": 500,
   "failed": 0,
   "acc": 0.382,
   "macro_f1": 0.3445,
   "nll": 1.3013,
   "brier": 0.7078,
   "ece": 0.0993,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6687,
   "acc_at_50cov": 0.52,
   "score_mae": 0.7684,
   "kl_gold": 0.6035,
   "soft_acc": 0.3499
  },
  "workflow:invoice_processing": {
   "n": 500,
   "failed": 0,
   "acc": 0.364,
   "macro_f1": 0.2195,
   "nll": 1.3258,
   "brier": 0.785,
   "ece": 0.2241,
   "p_true_zero": 0.002,
   "conf_auroc": 0.4117,
   "acc_at_50cov": 0.3,
   "score_mae": 1.0665,
   "kl_gold": 0.7698,
   "soft_acc": 0.3435
  },
  "workflow:security_incidents": {
   "n": 500,
   "failed": 0,
   "acc": 0.314,
   "macro_f1": 0.2096,
   "nll": 1.3299,
   "brier": 0.7574,
   "ece": 0.2311,
   "p_true_zero": 0.0,
   "conf_auroc": 0.695,
   "acc_at_50cov": 0.428,
   "score_mae": 0.97,
   "kl_gold": 0.4304,
   "soft_acc": 0.3277
  },
  "all": {
   "n": 4850,
   "failed": 0,
   "acc": 0.5095,
   "macro_f1": 0.3825,
   "nll": 1.5686,
   "brier": 0.6631,
   "ece": 0.2004,
   "p_true_zero": 0.0167,
   "conf_auroc": 0.7142,
   "acc_at_50cov": 0.668,
   "score_mae": 0.9823,
   "kl_gold": 0.5688,
   "soft_acc": 0.3487
  },
  "cost": {
   "usd": null,
   "input_tokens": 1003736,
   "output_tokens": 0,
   "calls": 3250
  },
  "latency_ms": {
   "p50": 74.9,
   "p95": 1213.4
  }
 },
 "laya_ara": {
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 0,
   "acc": 0.775,
   "macro_f1": 0.7731,
   "nll": 1.1416,
   "brier": 0.3925,
   "ece": 0.1612,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7966,
   "acc_at_50cov": 0.94
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.695,
   "macro_f1": 0.6461,
   "nll": 1.4526,
   "brier": 0.5141,
   "ece": 0.2183,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8309,
   "acc_at_50cov": 0.89
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.745,
   "macro_f1": 0.7403,
   "nll": 0.543,
   "brier": 0.3487,
   "ece": 0.0924,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7371,
   "acc_at_50cov": 0.87
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 0,
   "acc": 0.32,
   "macro_f1": 0.3226,
   "nll": 2.1354,
   "brier": 0.9172,
   "ece": 0.264,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6074,
   "acc_at_50cov": 0.38
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.885,
   "macro_f1": 0.8311,
   "nll": 0.4111,
   "brier": 0.205,
   "ece": 0.0914,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7405,
   "acc_at_50cov": 0.95
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.49,
   "macro_f1": 0.4967,
   "nll": 1.2405,
   "brier": 0.6951,
   "ece": 0.2537,
   "p_true_zero": 0.0,
   "conf_auroc": 0.6093,
   "acc_at_50cov": 0.57
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.7533,
   "macro_f1": 0.7479,
   "nll": 0.8284,
   "brier": 0.4331,
   "ece": 0.1857,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7173,
   "acc_at_50cov": 0.88
  },
  "type:choice": {
   "n": 950,
   "failed": 0,
   "acc": 0.5989,
   "macro_f1": 0.6547,
   "nll": 1.3877,
   "brier": 0.5987,
   "ece": 0.2085,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7747,
   "acc_at_50cov": 0.8253
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.815,
   "macro_f1": 0.806,
   "nll": 0.477,
   "brier": 0.2768,
   "ece": 0.0896,
   "p_true_zero": 0.0,
   "conf_auroc": 0.762,
   "acc_at_50cov": 0.93
  },
  "all": {
   "n": 1350,
   "failed": 0,
   "acc": 0.663,
   "macro_f1": 0.6693,
   "nll": 1.1178,
   "brier": 0.5033,
   "ece": 0.1733,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7846,
   "acc_at_50cov": 0.8622
  },
  "cost": {
   "usd": null,
   "input_tokens": 206423,
   "output_tokens": 0,
   "calls": 1350
  },
  "latency_ms": {
   "p50": 54.2,
   "p95": 124.8
  }
 },
 "laya_ara-rag": {
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 0,
   "acc": 0.55,
   "macro_f1": 0.542,
   "nll": 1.8302,
   "brier": 0.6539,
   "ece": 0.235,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7697,
   "acc_at_50cov": 0.73
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.605,
   "macro_f1": 0.5563,
   "nll": 1.873,
   "brier": 0.6165,
   "ece": 0.2229,
   "p_true_zero": 0.005,
   "conf_auroc": 0.7551,
   "acc_at_50cov": 0.77
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.865,
   "macro_f1": 0.864,
   "nll": 0.3197,
   "brier": 0.1957,
   "ece": 0.0365,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8132,
   "acc_at_50cov": 0.97
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 0,
   "acc": 0.715,
   "macro_f1": 0.7137,
   "nll": 0.8681,
   "brier": 0.3996,
   "ece": 0.1132,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8293,
   "acc_at_50cov": 0.92
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.845,
   "macro_f1": 0.6918,
   "nll": 0.3638,
   "brier": 0.2237,
   "ece": 0.0503,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7755,
   "acc_at_50cov": 0.95
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.44,
   "macro_f1": 0.3999,
   "nll": 1.0351,
   "brier": 0.6504,
   "ece": 0.1704,
   "p_true_zero": 0.0,
   "conf_auroc": 0.5502,
   "acc_at_50cov": 0.48
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.7,
   "macro_f1": 0.696,
   "nll": 0.7491,
   "brier": 0.4216,
   "ece": 0.087,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7692,
   "acc_at_50cov": 0.8667
  },
  "type:choice": {
   "n": 950,
   "failed": 0,
   "acc": 0.5968,
   "macro_f1": 0.6091,
   "nll": 1.2986,
   "brier": 0.5551,
   "ece": 0.1222,
   "p_true_zero": 0.0011,
   "conf_auroc": 0.7119,
   "acc_at_50cov": 0.7705
  },
  "type:noul": {
   "n": 400,
   "failed": 0,
   "acc": 0.855,
   "macro_f1": 0.8282,
   "nll": 0.3418,
   "brier": 0.2097,
   "ece": 0.0277,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7971,
   "acc_at_50cov": 0.965
  },
  "all": {
   "n": 1350,
   "failed": 0,
   "acc": 0.6733,
   "macro_f1": 0.6235,
   "nll": 1.0151,
   "brier": 0.4527,
   "ece": 0.0859,
   "p_true_zero": 0.0007,
   "conf_auroc": 0.7464,
   "acc_at_50cov": 0.8504
  },
  "cost": {
   "usd": null,
   "input_tokens": 206423,
   "output_tokens": 0,
   "calls": 1350
  },
  "latency_ms": {
   "p50": 57.9,
   "p95": 127.4
  }
 },
 "jev": {
  "suite:ag_news": {
   "n": 200,
   "failed": 0,
   "acc": 0.875,
   "macro_f1": 0.8706,
   "nll": 0.7855,
   "brier": 0.1891,
   "ece": 0.0917,
   "p_true_zero": 0.045,
   "conf_auroc": 0.8241,
   "acc_at_50cov": 0.96
  },
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 0,
   "acc": 0.83,
   "macro_f1": 0.8277,
   "nll": 0.672,
   "brier": 0.2202,
   "ece": 0.0524,
   "p_true_zero": 0.02,
   "conf_auroc": 0.8945,
   "acc_at_50cov": 0.98
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.635,
   "macro_f1": 0.5763,
   "nll": 1.9055,
   "brier": 0.5477,
   "ece": 0.2039,
   "p_true_zero": 0.075,
   "conf_auroc": 0.8083,
   "acc_at_50cov": 0.83
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.905,
   "macro_f1": 0.905,
   "nll": 0.2604,
   "brier": 0.1497,
   "ece": 0.0167,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8011,
   "acc_at_50cov": 0.97
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 0,
   "acc": 0.875,
   "macro_f1": 0.8695,
   "nll": 0.477,
   "brier": 0.1919,
   "ece": 0.0719,
   "p_true_zero": 0.01,
   "conf_auroc": 0.8559,
   "acc_at_50cov": 0.99
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.895,
   "macro_f1": 0.8571,
   "nll": 0.2342,
   "brier": 0.1457,
   "ece": 0.0513,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8969,
   "acc_at_50cov": 1.0
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.76,
   "macro_f1": 0.7564,
   "nll": 0.8425,
   "brier": 0.357,
   "ece": 0.0975,
   "p_true_zero": 0.02,
   "conf_auroc": 0.7706,
   "acc_at_50cov": 0.91
  },
  "suite:banking77": {
   "n": 200,
   "failed": 0,
   "acc": 0.795,
   "macro_f1": 0.7532,
   "nll": 1.2553,
   "brier": 0.2969,
   "ece": 0.0982,
   "p_true_zero": 0.065,
   "conf_auroc": 0.8791,
   "acc_at_50cov": 0.97
  },
  "suite:boolq": {
   "n": 200,
   "failed": 0,
   "acc": 0.895,
   "macro_f1": 0.8913,
   "nll": 0.2218,
   "brier": 0.1357,
   "ece": 0.0369,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9033,
   "acc_at_50cov": 0.99
  },
  "suite:emotion": {
   "n": 200,
   "failed": 0,
   "acc": 0.565,
   "macro_f1": 0.4834,
   "nll": 2.9014,
   "brier": 0.7014,
   "ece": 0.2999,
   "p_true_zero": 0.15,
   "conf_auroc": 0.7208,
   "acc_at_50cov": 0.74
  },
  "suite:sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.945,
   "nll": 0.1848,
   "brier": 0.0972,
   "ece": 0.0654,
   "p_true_zero": 0.0,
   "conf_auroc": 0.879,
   "acc_at_50cov": 1.0
  },
  "suite:sst5": {
   "n": 200,
   "failed": 0,
   "acc": 0.635,
   "macro_f1": 0.6204,
   "nll": 1.1167,
   "brier": 0.5628,
   "ece": 0.1811,
   "p_true_zero": 0.015,
   "conf_auroc": 0.5413,
   "acc_at_50cov": 0.69,
   "score_mae": 0.4424
  },
  "suite:typed-decisions": {
   "n": 2000,
   "failed": 0,
   "acc": 0.731,
   "macro_f1": 0.737,
   "nll": 0.6896,
   "brier": 0.3719,
   "ece": 0.0432,
   "p_true_zero": 0.0045,
   "conf_auroc": 0.7452,
   "acc_at_50cov": 0.88,
   "score_mae": 0.4007,
   "kl_gold": 0.8224,
   "soft_acc": 0.5836
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.6933,
   "macro_f1": 0.6978,
   "nll": 0.8241,
   "brier": 0.4294,
   "ece": 0.1427,
   "p_true_zero": 0.0067,
   "conf_auroc": 0.7262,
   "acc_at_50cov": 0.8267
  },
  "suite:xnli_en": {
   "n": 150,
   "failed": 0,
   "acc": 0.88,
   "macro_f1": 0.8783,
   "nll": 0.4722,
   "brier": 0.2018,
   "ece": 0.0716,
   "p_true_zero": 0.0133,
   "conf_auroc": 0.7729,
   "acc_at_50cov": 0.96
  },
  "suite:xnli_fr": {
   "n": 150,
   "failed": 0,
   "acc": 0.7667,
   "macro_f1": 0.7671,
   "nll": 0.6605,
   "brier": 0.3404,
   "ece": 0.1216,
   "p_true_zero": 0.0067,
   "conf_auroc": 0.7994,
   "acc_at_50cov": 0.9067
  },
  "type:choice": {
   "n": 2450,
   "failed": 0,
   "acc": 0.7559,
   "macro_f1": 0.7698,
   "nll": 0.9996,
   "brier": 0.3526,
   "ece": 0.0884,
   "p_true_zero": 0.0335,
   "conf_auroc": 0.7982,
   "acc_at_50cov": 0.9176,
   "kl_gold": 0.7477,
   "soft_acc": 0.5558
  },
  "type:noul": {
   "n": 1400,
   "failed": 0,
   "acc": 0.8579,
   "macro_f1": 0.8576,
   "nll": 0.3161,
   "brier": 0.1955,
   "ece": 0.0443,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8501,
   "acc_at_50cov": 0.98,
   "kl_gold": 0.0943,
   "soft_acc": 0.7072
  },
  "type:score": {
   "n": 1000,
   "failed": 0,
   "acc": 0.682,
   "macro_f1": 0.6651,
   "nll": 0.9525,
   "brier": 0.471,
   "ece": 0.1204,
   "p_true_zero": 0.011,
   "conf_auroc": 0.6813,
   "acc_at_50cov": 0.79,
   "score_mae": 0.409,
   "kl_gold": 1.4245,
   "soft_acc": 0.5116
  },
  "workflow:agent_trace_observability": {
   "n": 500,
   "failed": 0,
   "acc": 0.622,
   "macro_f1": 0.6043,
   "nll": 1.0323,
   "brier": 0.5197,
   "ece": 0.101,
   "p_true_zero": 0.008,
   "conf_auroc": 0.6711,
   "acc_at_50cov": 0.74,
   "score_mae": 0.5844,
   "kl_gold": 0.8871,
   "soft_acc": 0.4907
  },
  "workflow:customer_service": {
   "n": 500,
   "failed": 0,
   "acc": 0.79,
   "macro_f1": 0.7865,
   "nll": 0.4457,
   "brier": 0.2783,
   "ece": 0.0236,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8162,
   "acc_at_50cov": 0.956,
   "score_mae": 0.1939,
   "kl_gold": 0.8455,
   "soft_acc": 0.6244
  },
  "workflow:invoice_processing": {
   "n": 500,
   "failed": 0,
   "acc": 0.77,
   "macro_f1": 0.7218,
   "nll": 0.5326,
   "brier": 0.3056,
   "ece": 0.103,
   "p_true_zero": 0.0,
   "conf_auroc": 0.84,
   "acc_at_50cov": 0.96,
   "score_mae": 0.4404,
   "kl_gold": 0.3689,
   "soft_acc": 0.6795
  },
  "workflow:security_incidents": {
   "n": 500,
   "failed": 0,
   "acc": 0.742,
   "macro_f1": 0.7417,
   "nll": 0.7479,
   "brier": 0.3839,
   "ece": 0.0661,
   "p_true_zero": 0.01,
   "conf_auroc": 0.6784,
   "acc_at_50cov": 0.844,
   "score_mae": 0.384,
   "kl_gold": 1.1881,
   "soft_acc": 0.5398
  },
  "all": {
   "n": 4850,
   "failed": 0,
   "acc": 0.7701,
   "macro_f1": 0.7704,
   "nll": 0.7926,
   "brier": 0.3316,
   "ece": 0.0577,
   "p_true_zero": 0.0192,
   "conf_auroc": 0.7726,
   "acc_at_50cov": 0.9093,
   "score_mae": 0.409,
   "kl_gold": 0.8224,
   "soft_acc": 0.5836
  },
  "cost": {
   "usd": null,
   "input_tokens": 2157638,
   "output_tokens": 374716,
   "calls": 3250
  },
  "latency_ms": {
   "p50": 417.5,
   "p95": 850.7
  }
 },
 "or_jev": {
  "suite:ag_news": {
   "n": 200,
   "failed": 0,
   "acc": 0.885,
   "macro_f1": 0.8812,
   "nll": 0.7828,
   "brier": 0.187,
   "ece": 0.0856,
   "p_true_zero": 0.045,
   "conf_auroc": 0.8015,
   "acc_at_50cov": 0.95
  },
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 0,
   "acc": 0.84,
   "macro_f1": 0.8356,
   "nll": 0.7217,
   "brier": 0.2209,
   "ece": 0.0346,
   "p_true_zero": 0.025,
   "conf_auroc": 0.8784,
   "acc_at_50cov": 0.98
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.63,
   "macro_f1": 0.5726,
   "nll": 1.9594,
   "brier": 0.5483,
   "ece": 0.2101,
   "p_true_zero": 0.08,
   "conf_auroc": 0.8165,
   "acc_at_50cov": 0.85
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.905,
   "macro_f1": 0.905,
   "nll": 0.2648,
   "brier": 0.1534,
   "ece": 0.027,
   "p_true_zero": 0.0,
   "conf_auroc": 0.7922,
   "acc_at_50cov": 0.97
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 0,
   "acc": 0.865,
   "macro_f1": 0.8584,
   "nll": 0.5279,
   "brier": 0.1964,
   "ece": 0.0606,
   "p_true_zero": 0.015,
   "conf_auroc": 0.8679,
   "acc_at_50cov": 0.98
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.9,
   "macro_f1": 0.8629,
   "nll": 0.2367,
   "brier": 0.1475,
   "ece": 0.0554,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8817,
   "acc_at_50cov": 1.0
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.76,
   "macro_f1": 0.7563,
   "nll": 0.88,
   "brier": 0.3553,
   "ece": 0.0966,
   "p_true_zero": 0.025,
   "conf_auroc": 0.7717,
   "acc_at_50cov": 0.9
  },
  "suite:banking77": {
   "n": 200,
   "failed": 0,
   "acc": 0.79,
   "macro_f1": 0.7506,
   "nll": 1.2486,
   "brier": 0.2941,
   "ece": 0.1013,
   "p_true_zero": 0.065,
   "conf_auroc": 0.8887,
   "acc_at_50cov": 0.97
  },
  "suite:boolq": {
   "n": 200,
   "failed": 0,
   "acc": 0.895,
   "macro_f1": 0.8913,
   "nll": 0.2228,
   "brier": 0.1377,
   "ece": 0.0392,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9046,
   "acc_at_50cov": 0.99
  },
  "suite:emotion": {
   "n": 200,
   "failed": 0,
   "acc": 0.56,
   "macro_f1": 0.4764,
   "nll": 2.9004,
   "brier": 0.7042,
   "ece": 0.3039,
   "p_true_zero": 0.15,
   "conf_auroc": 0.7286,
   "acc_at_50cov": 0.74
  },
  "suite:sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.945,
   "macro_f1": 0.945,
   "nll": 0.183,
   "brier": 0.0959,
   "ece": 0.0644,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8773,
   "acc_at_50cov": 0.99
  },
  "suite:sst5": {
   "n": 200,
   "failed": 0,
   "acc": 0.62,
   "macro_f1": 0.6065,
   "nll": 1.1204,
   "brier": 0.5649,
   "ece": 0.1605,
   "p_true_zero": 0.015,
   "conf_auroc": 0.5664,
   "acc_at_50cov": 0.68,
   "score_mae": 0.4431
  },
  "suite:typed-decisions": {
   "n": 2000,
   "failed": 0,
   "acc": 0.7265,
   "macro_f1": 0.7288,
   "nll": 0.6772,
   "brier": 0.3722,
   "ece": 0.0398,
   "p_true_zero": 0.003,
   "conf_auroc": 0.7516,
   "acc_at_50cov": 0.88,
   "score_mae": 0.4013,
   "kl_gold": 0.8087,
   "soft_acc": 0.5828
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.6933,
   "macro_f1": 0.6978,
   "nll": 0.7931,
   "brier": 0.4212,
   "ece": 0.1398,
   "p_true_zero": 0.0067,
   "conf_auroc": 0.7379,
   "acc_at_50cov": 0.8267
  },
  "suite:xnli_en": {
   "n": 150,
   "failed": 0,
   "acc": 0.86,
   "macro_f1": 0.8571,
   "nll": 0.4193,
   "brier": 0.2077,
   "ece": 0.0822,
   "p_true_zero": 0.0067,
   "conf_auroc": 0.8293,
   "acc_at_50cov": 0.96
  },
  "suite:xnli_fr": {
   "n": 150,
   "failed": 0,
   "acc": 0.7667,
   "macro_f1": 0.7671,
   "nll": 0.7184,
   "brier": 0.3428,
   "ece": 0.1224,
   "p_true_zero": 0.0133,
   "conf_auroc": 0.7948,
   "acc_at_50cov": 0.9067
  },
  "type:choice": {
   "n": 2450,
   "failed": 0,
   "acc": 0.7539,
   "macro_f1": 0.7681,
   "nll": 1.0124,
   "brier": 0.3527,
   "ece": 0.0904,
   "p_true_zero": 0.0351,
   "conf_auroc": 0.8006,
   "acc_at_50cov": 0.9176,
   "kl_gold": 0.7269,
   "soft_acc": 0.5566
  },
  "type:noul": {
   "n": 1400,
   "failed": 0,
   "acc": 0.8557,
   "macro_f1": 0.8554,
   "nll": 0.3167,
   "brier": 0.1961,
   "ece": 0.0431,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8523,
   "acc_at_50cov": 0.9829,
   "kl_gold": 0.0938,
   "soft_acc": 0.7051
  },
  "type:score": {
   "n": 1000,
   "failed": 0,
   "acc": 0.675,
   "macro_f1": 0.6567,
   "nll": 0.93,
   "brier": 0.4726,
   "ece": 0.1224,
   "p_true_zero": 0.008,
   "conf_auroc": 0.6917,
   "acc_at_50cov": 0.794,
   "score_mae": 0.4097,
   "kl_gold": 1.4063,
   "soft_acc": 0.5108
  },
  "workflow:agent_trace_observability": {
   "n": 500,
   "failed": 0,
   "acc": 0.618,
   "macro_f1": 0.5998,
   "nll": 1.0007,
   "brier": 0.5211,
   "ece": 0.1047,
   "p_true_zero": 0.004,
   "conf_auroc": 0.6725,
   "acc_at_50cov": 0.74,
   "score_mae": 0.584,
   "kl_gold": 0.86,
   "soft_acc": 0.4901
  },
  "workflow:customer_service": {
   "n": 500,
   "failed": 0,
   "acc": 0.774,
   "macro_f1": 0.7704,
   "nll": 0.4443,
   "brier": 0.2783,
   "ece": 0.0383,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8396,
   "acc_at_50cov": 0.96,
   "score_mae": 0.1952,
   "kl_gold": 0.8411,
   "soft_acc": 0.6211
  },
  "workflow:invoice_processing": {
   "n": 500,
   "failed": 0,
   "acc": 0.77,
   "macro_f1": 0.7193,
   "nll": 0.5362,
   "brier": 0.3072,
   "ece": 0.0766,
   "p_true_zero": 0.0,
   "conf_auroc": 0.8373,
   "acc_at_50cov": 0.96,
   "score_mae": 0.4432,
   "kl_gold": 0.3745,
   "soft_acc": 0.679
  },
  "workflow:security_incidents": {
   "n": 500,
   "failed": 0,
   "acc": 0.744,
   "macro_f1": 0.7388,
   "nll": 0.7276,
   "brier": 0.3822,
   "ece": 0.0786,
   "p_true_zero": 0.008,
   "conf_auroc": 0.6791,
   "acc_at_50cov": 0.832,
   "score_mae": 0.3828,
   "kl_gold": 1.1592,
   "soft_acc": 0.5411
  },
  "all": {
   "n": 4850,
   "failed": 0,
   "acc": 0.767,
   "macro_f1": 0.7686,
   "nll": 0.7946,
   "brier": 0.3322,
   "ece": 0.061,
   "p_true_zero": 0.0194,
   "conf_auroc": 0.7769,
   "acc_at_50cov": 0.9101,
   "score_mae": 0.4097,
   "kl_gold": 0.8087,
   "soft_acc": 0.5828
  },
  "cost": {
   "usd": 0.0906,
   "input_tokens": 2157160,
   "output_tokens": 374716,
   "calls": 3250
  },
  "latency_ms": {
   "p50": 372.4,
   "p95": 469.1
  }
 },
 "luna": {
  "suite:ag_news": {
   "n": 200,
   "failed": 0,
   "acc": 0.875,
   "macro_f1": 0.8699,
   "nll": 1.0269,
   "brier": 0.226,
   "ece": 0.1122,
   "p_true_zero": 0.06,
   "conf_auroc": 0.7302,
   "acc_at_50cov": 0.94
  },
  "suite:ar_massive_intent": {
   "n": 200,
   "failed": 1,
   "acc": 0.845,
   "macro_f1": 0.8435,
   "nll": 0.6113,
   "brier": 0.2161,
   "ece": 0.0511,
   "p_true_zero": 0.01,
   "conf_auroc": 0.8883,
   "acc_at_50cov": 0.99
  },
  "suite:ar_massive_scenario": {
   "n": 200,
   "failed": 0,
   "acc": 0.66,
   "macro_f1": 0.6428,
   "nll": 1.5683,
   "brier": 0.4996,
   "ece": 0.1682,
   "p_true_zero": 0.05,
   "conf_auroc": 0.7626,
   "acc_at_50cov": 0.81
  },
  "suite:ar_miracl_pair": {
   "n": 200,
   "failed": 0,
   "acc": 0.875,
   "macro_f1": 0.8746,
   "nll": 0.9822,
   "brier": 0.2121,
   "ece": 0.1069,
   "p_true_zero": 0.06,
   "conf_auroc": 0.7183,
   "acc_at_50cov": 0.93
  },
  "suite:ar_miracl_rerank": {
   "n": 200,
   "failed": 1,
   "acc": 0.86,
   "macro_f1": 0.8564,
   "nll": 0.5849,
   "brier": 0.1851,
   "ece": 0.0445,
   "p_true_zero": 0.02,
   "conf_auroc": 0.8874,
   "acc_at_50cov": 0.97
  },
  "suite:ar_offensive": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.9075,
   "nll": 0.1346,
   "brier": 0.0846,
   "ece": 0.0218,
   "p_true_zero": 0.0,
   "conf_auroc": 0.9513,
   "acc_at_50cov": 1.0
  },
  "suite:ar_sentiment": {
   "n": 200,
   "failed": 0,
   "acc": 0.8,
   "macro_f1": 0.797,
   "nll": 0.7841,
   "brier": 0.3228,
   "ece": 0.107,
   "p_true_zero": 0.02,
   "conf_auroc": 0.7727,
   "acc_at_50cov": 0.95
  },
  "suite:banking77": {
   "n": 200,
   "failed": 0,
   "acc": 0.775,
   "macro_f1": 0.7357,
   "nll": 1.3444,
   "brier": 0.3297,
   "ece": 0.087,
   "p_true_zero": 0.06,
   "conf_auroc": 0.8308,
   "acc_at_50cov": 0.95
  },
  "suite:boolq": {
   "n": 200,
   "failed": 0,
   "acc": 0.855,
   "macro_f1": 0.848,
   "nll": 0.6244,
   "brier": 0.2268,
   "ece": 0.0859,
   "p_true_zero": 0.025,
   "conf_auroc": 0.7973,
   "acc_at_50cov": 0.96
  },
  "suite:emotion": {
   "n": 200,
   "failed": 0,
   "acc": 0.585,
   "macro_f1": 0.4942,
   "nll": 2.1635,
   "brier": 0.6595,
   "ece": 0.2609,
   "p_true_zero": 0.08,
   "conf_auroc": 0.7158,
   "acc_at_50cov": 0.73
  },
  "suite:sst2": {
   "n": 200,
   "failed": 0,
   "acc": 0.935,
   "macro_f1": 0.9349,
   "nll": 0.3363,
   "brier": 0.1268,
   "ece": 0.0482,
   "p_true_zero": 0.01,
   "conf_auroc": 0.6919,
   "acc_at_50cov": 0.96
  },
  "suite:sst5": {
   "n": 200,
   "failed": 0,
   "acc": 0.615,
   "macro_f1": 0.5874,
   "nll": 1.1745,
   "brier": 0.5806,
   "ece": 0.1598,
   "p_true_zero": 0.015,
   "conf_auroc": 0.5293,
   "acc_at_50cov": 0.62,
   "score_mae": 0.4705
  },
  "suite:typed-decisions": {
   "n": 2000,
   "failed": 0,
   "acc": 0.7105,
   "macro_f1": 0.7158,
   "nll": 0.9195,
   "brier": 0.4088,
   "ece": 0.0842,
   "p_true_zero": 0.017,
   "conf_auroc": 0.751,
   "acc_at_50cov": 0.871,
   "score_mae": 0.3965,
   "kl_gold": 1.0731,
   "soft_acc": 0.5766
  },
  "suite:xnli_ar": {
   "n": 150,
   "failed": 0,
   "acc": 0.54,
   "macro_f1": 0.5191,
   "nll": 2.0349,
   "brier": 0.7334,
   "ece": 0.2983,
   "p_true_zero": 0.0733,
   "conf_auroc": 0.5596,
   "acc_at_50cov": 0.5867
  },
  "suite:xnli_en": {
   "n": 150,
   "failed": 0,
   "acc": 0.6667,
   "macro_f1": 0.6754,
   "nll": 1.0596,
   "brier": 0.5059,
   "ece": 0.2069,
   "p_true_zero": 0.02,
   "conf_auroc": 0.6761,
   "acc_at_50cov": 0.8133
  },
  "suite:xnli_fr": {
   "n": 150,
   "failed": 0,
   "acc": 0.62,
   "macro_f1": 0.6197,
   "nll": 1.3828,
   "brier": 0.5803,
   "ece": 0.2151,
   "p_true_zero": 0.0333,
   "conf_auroc": 0.6002,
   "acc_at_50cov": 0.6933
  },
  "type:choice": {
   "n": 2450,
   "failed": 2,
   "acc": 0.7135,
   "macro_f1": 0.7586,
   "nll": 1.255,
   "brier": 0.4321,
   "ece": 0.1323,
   "p_true_zero": 0.04,
   "conf_auroc": 0.76,
   "acc_at_50cov": 0.8686,
   "kl_gold": 1.5442,
   "soft_acc": 0.5205
  },
  "type:noul": {
   "n": 1400,
   "failed": 0,
   "acc": 0.8564,
   "macro_f1": 0.8564,
   "nll": 0.502,
   "brier": 0.2006,
   "ece": 0.0429,
   "p_true_zero": 0.0171,
   "conf_auroc": 0.7978,
   "acc_at_50cov": 0.95,
   "kl_gold": 0.3544,
   "soft_acc": 0.7108
  },
  "type:score": {
   "n": 1000,
   "failed": 0,
   "acc": 0.671,
   "macro_f1": 0.6509,
   "nll": 1.0002,
   "brier": 0.485,
   "ece": 0.1191,
   "p_true_zero": 0.013,
   "conf_auroc": 0.675,
   "acc_at_50cov": 0.776,
   "score_mae": 0.4113,
   "kl_gold": 1.2587,
   "soft_acc": 0.5181
  },
  "workflow:agent_trace_observability": {
   "n": 500,
   "failed": 0,
   "acc": 0.536,
   "macro_f1": 0.5055,
   "nll": 1.3279,
   "brier": 0.6174,
   "ece": 0.1798,
   "p_true_zero": 0.014,
   "conf_auroc": 0.7006,
   "acc_at_50cov": 0.692,
   "score_mae": 0.5286,
   "kl_gold": 0.9593,
   "soft_acc": 0.4508
  },
  "workflow:customer_service": {
   "n": 500,
   "failed": 0,
   "acc": 0.796,
   "macro_f1": 0.7878,
   "nll": 0.8267,
   "brier": 0.3227,
   "ece": 0.0813,
   "p_true_zero": 0.024,
   "conf_auroc": 0.7371,
   "acc_at_50cov": 0.892,
   "score_mae": 0.267,
   "kl_gold": 1.4477,
   "soft_acc": 0.6346
  },
  "workflow:invoice_processing": {
   "n": 500,
   "failed": 0,
   "acc": 0.802,
   "macro_f1": 0.7676,
   "nll": 0.5976,
   "brier": 0.282,
   "ece": 0.0466,
   "p_true_zero": 0.008,
   "conf_auroc": 0.8169,
   "acc_at_50cov": 0.972,
   "score_mae": 0.3849,
   "kl_gold": 0.451,
   "soft_acc": 0.696
  },
  "workflow:security_incidents": {
   "n": 500,
   "failed": 0,
   "acc": 0.708,
   "macro_f1": 0.6966,
   "nll": 0.9256,
   "brier": 0.4132,
   "ece": 0.1084,
   "p_true_zero": 0.022,
   "conf_auroc": 0.7167,
   "acc_at_50cov": 0.836,
   "score_mae": 0.4056,
   "kl_gold": 1.4342,
   "soft_acc": 0.5251
  },
  "all": {
   "n": 4850,
   "failed": 2,
   "acc": 0.746,
   "macro_f1": 0.7616,
   "nll": 0.9851,
   "brier": 0.3762,
   "ece": 0.0978,
   "p_true_zero": 0.0278,
   "conf_auroc": 0.7609,
   "acc_at_50cov": 0.8907,
   "score_mae": 0.4113,
   "kl_gold": 1.0731,
   "soft_acc": 0.5766
  },
  "cost": {
   "usd": 0.1308,
   "input_tokens": 1308453,
   "output_tokens": 0,
   "calls": 3248
  },
  "latency_ms": {
   "p50": 362.8,
   "p95": 460.1
  }
 }
}