{
 "claude-opus-5-5": {
  "means": {
   "bare": 2.4697317460317456,
   "show_all": 1.696895238095238,
   "bare2": 3.3247031746031745,
   "attempt_then_consult": 3.0295698412698413,
   "consult_on_failure": 3.269188888888889,
   "audit_then_consult": 3.1254873015873015,
   "prior_then_consult": 3.30433492063492
  },
  "contrasts": {
   "PRIMARY attempt_then_consult - bare2": {
    "mean": -0.29513333333333325,
    "lo": -0.7056349206349205,
    "hi": 0.13029841269841252,
    "up": 4,
    "down": 17
   },
   "audit_then_consult - bare2": {
    "mean": -0.199215873015873,
    "lo": -0.5410428571428572,
    "hi": 0.15603333333333338,
    "up": 7,
    "down": 14
   },
   "consult_on_failure - bare2": {
    "mean": -0.055514285714285704,
    "lo": -0.11951746031746027,
    "hi": -0.005887301587301584,
    "up": 0,
    "down": 4
   },
   "attempt_then_consult - prior_then_consult": {
    "mean": -0.27476507936507916,
    "lo": -0.5785619047619047,
    "hi": -0.019331746031745606,
    "up": 6,
    "down": 15
   },
   "audit_then_consult - attempt_then_consult": {
    "mean": 0.09591746031746025,
    "lo": -0.10462063492063513,
    "hi": 0.285295238095238,
    "up": 14,
    "down": 6
   },
   "prior_then_consult - bare2": {
    "mean": -0.020368253968254105,
    "lo": -0.40107142857142886,
    "hi": 0.2961936507936506,
    "up": 8,
    "down": 13
   },
   "show_all - bare": {
    "mean": -0.7728365079365078,
    "lo": -1.5286460317460315,
    "hi": -0.19050793650793654,
    "up": 6,
    "down": 15
   },
   "bare2 - bare": {
    "mean": 0.8549714285714286,
    "lo": 0.500195238095238,
    "hi": 1.2607873015873017,
    "up": 18,
    "down": 1
   },
   "attempt_then_consult - bare": {
    "mean": 0.5598380952380954,
    "lo": 0.27633968253968255,
    "hi": 0.9588777777777779,
    "up": 16,
    "down": 4
   },
   "audit_then_consult - bare": {
    "mean": 0.6557555555555556,
    "lo": 0.38007460317460323,
    "hi": 0.9840238095238095,
    "up": 18,
    "down": 1
   },
   "consult_on_failure - bare": {
    "mean": 0.799457142857143,
    "lo": 0.4667063492063493,
    "hi": 1.1843920634920635,
    "up": 18,
    "down": 1
   },
   "prior_then_consult - bare": {
    "mean": 0.8346031746031745,
    "lo": 0.5000079365079365,
    "hi": 1.2245031746031745,
    "up": 19,
    "down": 2
   },
   "attempt_then_consult - show_all": {
    "mean": 1.3326746031746033,
    "lo": 0.6525650793650793,
    "hi": 2.2177507936507936,
    "up": 20,
    "down": 1
   },
   "consult_on_failure - prior_then_consult": {
    "mean": -0.0351460317460316,
    "lo": -0.34676825396825367,
    "hi": 0.3405904761904764,
    "up": 10,
    "down": 11
   },
   "audit_then_consult - prior_then_consult": {
    "mean": -0.17884761904761892,
    "lo": -0.44294761904761903,
    "hi": 0.03278412698412704,
    "up": 10,
    "down": 11
   }
  },
  "diag": {
   "P2_consult - P2_refine (raw second pass)": {
    "mean": -0.19012222222222214,
    "lo": -0.6896730158730158,
    "hi": 0.2745904761904763
   },
   "P2_audit - P2_refine (raw second pass)": {
    "mean": -0.17139682539682521,
    "lo": -0.5465111111111111,
    "hi": 0.18886190476190517
   },
   "P2_prior - P2_refine (raw second pass)": {
    "mean": 0.03483650793650798,
    "lo": -0.39159206349206327,
    "hi": 0.3974809523809521
   },
   "S - P1 (raw pass evals)": {
    "mean": -0.7614555555555556,
    "lo": -1.526979365079365,
    "hi": -0.17234126984126985
   }
  }
 },
 "claude-opus-4-8": {
  "means": {
   "bare": 0.6254142857142858,
   "show_all": 1.1124238095238095,
   "bare2": 1.15455,
   "attempt_then_consult": 1.303942857142857,
   "consult_on_failure": 1.3105119047619047,
   "audit_then_consult": 1.3220023809523809,
   "prior_then_consult": 1.2486142857142857
  },
  "contrasts": {
   "PRIMARY attempt_then_consult - bare2": {
    "mean": 0.1493928571428572,
    "lo": -0.0752690476190476,
    "hi": 0.5035261904761905,
    "up": 12,
    "down": 9
   },
   "audit_then_consult - bare2": {
    "mean": 0.16745238095238094,
    "lo": -0.04464523809523809,
    "hi": 0.5197738095238096,
    "up": 13,
    "down": 8
   },
   "consult_on_failure - bare2": {
    "mean": 0.1559619047619048,
    "lo": -0.06366428571428569,
    "hi": 0.516157142857143,
    "up": 13,
    "down": 7
   },
   "attempt_then_consult - prior_then_consult": {
    "mean": 0.055328571428571414,
    "lo": -0.06122142857142857,
    "hi": 0.18782380952380953,
    "up": 12,
    "down": 9
   },
   "audit_then_consult - attempt_then_consult": {
    "mean": 0.018059523809523775,
    "lo": -0.04326428571428571,
    "hi": 0.09814285714285706,
    "up": 12,
    "down": 8
   },
   "prior_then_consult - bare2": {
    "mean": 0.09406428571428575,
    "lo": -0.038461904761904704,
    "hi": 0.32982857142857147,
    "up": 9,
    "down": 12
   },
   "show_all - bare": {
    "mean": 0.4870095238095238,
    "lo": 0.24855952380952379,
    "hi": 0.7156190476190476,
    "up": 18,
    "down": 3
   },
   "bare2 - bare": {
    "mean": 0.5291357142857143,
    "lo": 0.31894285714285714,
    "hi": 0.793102380952381,
    "up": 18,
    "down": 2
   },
   "attempt_then_consult - bare": {
    "mean": 0.6785285714285715,
    "lo": 0.3981261904761905,
    "hi": 1.028802380952381,
    "up": 20,
    "down": 1
   },
   "audit_then_consult - bare": {
    "mean": 0.6965880952380952,
    "lo": 0.3891880952380952,
    "hi": 1.0804761904761904,
    "up": 20,
    "down": 1
   },
   "consult_on_failure - bare": {
    "mean": 0.685097619047619,
    "lo": 0.3996047619047619,
    "hi": 1.0441500000000001,
    "up": 20,
    "down": 1
   },
   "prior_then_consult - bare": {
    "mean": 0.6232,
    "lo": 0.36784761904761903,
    "hi": 0.9353785714285714,
    "up": 18,
    "down": 1
   },
   "attempt_then_consult - show_all": {
    "mean": 0.1915190476190476,
    "lo": 0.020616666666666637,
    "hi": 0.40779523809523804,
    "up": 11,
    "down": 9
   },
   "consult_on_failure - prior_then_consult": {
    "mean": 0.06189761904761905,
    "lo": -0.050188095238095254,
    "hi": 0.19225952380952385,
    "up": 13,
    "down": 8
   },
   "audit_then_consult - prior_then_consult": {
    "mean": 0.0733880952380952,
    "lo": -0.02385238095238099,
    "hi": 0.1978666666666666,
    "up": 14,
    "down": 7
   }
  },
  "diag": {
   "P2_consult - P2_refine (raw second pass)": {
    "mean": 0.2128428571428571,
    "lo": -0.036547619047619065,
    "hi": 0.5831500000000001
   },
   "P2_audit - P2_refine (raw second pass)": {
    "mean": 0.11554761904761908,
    "lo": -0.2545166666666666,
    "hi": 0.5225857142857143
   },
   "P2_prior - P2_refine (raw second pass)": {
    "mean": 0.10983333333333337,
    "lo": -0.0888119047619047,
    "hi": 0.3799047619047619
   },
   "S - P1 (raw pass evals)": {
    "mean": 0.43700238095238103,
    "lo": 0.20757142857142857,
    "hi": 0.6660238095238096
   }
  }
 }
}