GUARD {"rows": 720, "infra_rows_excluded": 0, "duplicates_dropped": 0, "scored_ok_rows_with_other_fingerprint": 0} missing_cells: 0 [] { "bare": { "n": 78, "mean_score": 1.862, "fast1": 71, "correct": 77, "status": { "ok": 77, "incorrect": 1 }, "uses_bf16": 0, "uses_bf16_on_p1": 0, "n_on_p1": 42, "tokens_in_mean": 8378, "tokens_out_mean": 8065, "calls_mean": 2.22, "wall_mean_s": 160.8, "ctx_chars_mean": 0, "mean_pooled": 1.862, "mean_dev": 2.135, "mean_holdout": 1.692, "mean_p1_applicable": 1.255, "mean_engine_card": 1.907, "mean_no_p1_engine_card": 2.571, "mean_holdout_p1": 1.308, "mean_holdout_nonp1": 1.99, "mean_holdout_engine_card": 1.692, "mean_dev_p1": 1.202, "mean_holdout_ace_named": 1.545, "mean_holdout_ace_unnamed": 1.806, "mean_e5_problems": 1.684 }, "prior": { "n": 78, "mean_score": 1.956, "fast1": 72, "correct": 78, "status": { "ok": 78 }, "uses_bf16": 0, "uses_bf16_on_p1": 0, "n_on_p1": 42, "tokens_in_mean": 8725, "tokens_out_mean": 9476, "calls_mean": 2.19, "wall_mean_s": 180.7, "ctx_chars_mean": 495, "mean_pooled": 1.956, "mean_dev": 2.334, "mean_holdout": 1.72, "mean_p1_applicable": 1.432, "mean_engine_card": 1.942, "mean_no_p1_engine_card": 2.568, "mean_holdout_p1": 1.355, "mean_holdout_nonp1": 2.005, "mean_holdout_engine_card": 1.72, "mean_dev_p1": 1.508, "mean_holdout_ace_named": 1.595, "mean_holdout_ace_unnamed": 1.818, "mean_e5_problems": 1.571 }, "ace": { "n": 78, "mean_score": 1.954, "fast1": 74, "correct": 78, "status": { "ok": 78 }, "uses_bf16": 0, "uses_bf16_on_p1": 0, "n_on_p1": 42, "tokens_in_mean": 290871, "tokens_out_mean": 9946, "calls_mean": 2.17, "wall_mean_s": 226.4, "ctx_chars_mean": 305234, "mean_pooled": 1.954, "mean_dev": 2.522, "mean_holdout": 1.599, "mean_p1_applicable": 1.539, "mean_engine_card": 1.888, "mean_no_p1_engine_card": 2.437, "mean_holdout_p1": 1.253, "mean_holdout_nonp1": 1.867, "mean_holdout_engine_card": 1.599, "mean_dev_p1": 1.826, "mean_holdout_ace_named": 1.47, "mean_holdout_ace_unnamed": 1.698, "mean_e5_problems": 1.594 }, "aceseed": { "n": 78, "mean_score": 2.834, "fast1": 73, "correct": 74, "status": { "ok": 74, "incorrect": 4 }, "uses_bf16": 33, "uses_bf16_on_p1": 33, "n_on_p1": 42, "tokens_in_mean": 306265, "tokens_out_mean": 10380, "calls_mean": 2.24, "wall_mean_s": 223.9, "ctx_chars_mean": 309122, "mean_pooled": 2.834, "mean_dev": 3.716, "mean_holdout": 2.282, "mean_p1_applicable": 3.317, "mean_engine_card": 2.899, "mean_no_p1_engine_card": 2.27, "mean_holdout_p1": 2.858, "mean_holdout_nonp1": 1.835, "mean_holdout_engine_card": 2.282, "mean_dev_p1": 3.777, "mean_holdout_ace_named": 2.092, "mean_holdout_ace_unnamed": 2.431, "mean_e5_problems": 1.575 }, "stock": { "n": 78, "mean_score": 1.726, "fast1": 71, "correct": 78, "status": { "ok": 78 }, "uses_bf16": 0, "uses_bf16_on_p1": 0, "n_on_p1": 42, "tokens_in_mean": 13731, "tokens_out_mean": 6025, "calls_mean": 2.17, "wall_mean_s": 142.6, "ctx_chars_mean": 6982, "mean_pooled": 1.726, "mean_dev": 2.098, "mean_holdout": 1.493, "mean_p1_applicable": 1.256, "mean_engine_card": 1.762, "mean_no_p1_engine_card": 2.273, "mean_holdout_p1": 1.265, "mean_holdout_nonp1": 1.67, "mean_holdout_engine_card": 1.493, "mean_dev_p1": 1.248, "mean_holdout_ace_named": 1.452, "mean_holdout_ace_unnamed": 1.524, "mean_e5_problems": 1.515 }, "livP1": { "n": 78, "mean_score": 2.727, "fast1": 72, "correct": 74, "status": { "ok": 74, "incorrect": 4 }, "uses_bf16": 34, "uses_bf16_on_p1": 34, "n_on_p1": 42, "tokens_in_mean": 19454, "tokens_out_mean": 7542, "calls_mean": 2.22, "wall_mean_s": 153.8, "ctx_chars_mean": 10770, "mean_pooled": 2.727, "mean_dev": 3.891, "mean_holdout": 2.0, "mean_p1_applicable": 3.184, "mean_engine_card": 2.766, "mean_no_p1_engine_card": 2.194, "mean_holdout_p1": 2.622, "mean_holdout_nonp1": 1.516, "mean_holdout_engine_card": 2.0, "mean_dev_p1": 3.746, "mean_holdout_ace_named": 1.87, "mean_holdout_ace_unnamed": 2.101, "mean_e5_problems": 1.66 }, "livEng": { "n": 78, "mean_score": 3.352, "fast1": 69, "correct": 72, "status": { "ok": 72, "incorrect": 6 }, "uses_bf16": 65, "uses_bf16_on_p1": 37, "n_on_p1": 42, "tokens_in_mean": 25087, "tokens_out_mean": 10194, "calls_mean": 2.31, "wall_mean_s": 189.4, "ctx_chars_mean": 14671, "mean_pooled": 3.352, "mean_dev": 3.974, "mean_holdout": 2.964, "mean_p1_applicable": 3.108, "mean_engine_card": 3.461, "mean_no_p1_engine_card": 3.637, "mean_holdout_p1": 2.769, "mean_holdout_nonp1": 3.116, "mean_holdout_engine_card": 2.964, "mean_dev_p1": 3.448, "mean_holdout_ace_named": 2.443, "mean_holdout_ace_unnamed": 3.37, "mean_e5_problems": 1.953 }, "livKF": { "n": 78, "mean_score": 3.667, "fast1": 71, "correct": 74, "status": { "ok": 74, "incorrect": 4 }, "uses_bf16": 65, "uses_bf16_on_p1": 37, "n_on_p1": 42, "tokens_in_mean": 30073, "tokens_out_mean": 10038, "calls_mean": 2.32, "wall_mean_s": 182.7, "ctx_chars_mean": 18478, "mean_pooled": 3.667, "mean_dev": 4.293, "mean_holdout": 3.276, "mean_p1_applicable": 3.415, "mean_engine_card": 3.734, "mean_no_p1_engine_card": 3.961, "mean_holdout_p1": 3.129, "mean_holdout_nonp1": 3.39, "mean_holdout_engine_card": 3.276, "mean_dev_p1": 3.701, "mean_holdout_ace_named": 3.041, "mean_holdout_ace_unnamed": 3.458, "mean_e5_problems": 2.507 }, "aceKF": { "n": 78, "mean_score": 3.291, "fast1": 69, "correct": 73, "status": { "ok": 73, "incorrect": 5 }, "uses_bf16": 57, "uses_bf16_on_p1": 37, "n_on_p1": 42, "tokens_in_mean": 312458, "tokens_out_mean": 10040, "calls_mean": 2.23, "wall_mean_s": 185.6, "ctx_chars_mean": 316582, "mean_pooled": 3.291, "mean_dev": 3.676, "mean_holdout": 3.051, "mean_p1_applicable": 3.288, "mean_engine_card": 3.381, "mean_no_p1_engine_card": 3.295, "mean_holdout_p1": 2.924, "mean_holdout_nonp1": 3.149, "mean_holdout_engine_card": 3.051, "mean_dev_p1": 3.651, "mean_holdout_ace_named": 2.744, "mean_holdout_ace_unnamed": 3.29, "mean_e5_problems": 1.53 }, "livE5": { "n": 18, "mean_score": 1.633, "fast1": 16, "correct": 18, "status": { "ok": 18 }, "uses_bf16": 4, "uses_bf16_on_p1": 3, "n_on_p1": 6, "tokens_in_mean": 23402, "tokens_out_mean": 11967, "calls_mean": 2.06, "wall_mean_s": 178.4, "ctx_chars_mean": 14521, "mean_pooled": 1.633, "mean_dev": 1.823, "mean_holdout": 1.595, "mean_p1_applicable": 1.861, "mean_engine_card": 1.633, "mean_no_p1_engine_card": 1.519, "mean_holdout_p1": 1.861, "mean_holdout_nonp1": 1.418, "mean_holdout_engine_card": 1.595, "mean_dev_p1": null, "mean_holdout_ace_named": 1.861, "mean_holdout_ace_unnamed": 1.418, "mean_e5_problems": 1.633 } } livKF-bare|pooled n=26 +1.805 [+1.330, +2.280] 22:3 livKF-prior|pooled n=26 +1.711 [+1.225, +2.206] 22:3 livKF-ace|pooled n=26 +1.713 [+1.187, +2.248] 22:4 livKF-aceseed|pooled n=26 +0.833 [+0.327, +1.399] 17:7 livKF-stock|pooled n=26 +1.941 [+1.471, +2.422] 24:1 livKF-aceKF|pooled n=26 +0.376 [-0.028, +0.843] 13:11 livEng-stock|pooled n=26 +1.627 [+1.162, +2.104] 22:3 livEng-bare|pooled n=26 +1.490 [+1.041, +1.952] 21:3 livEng-prior|pooled n=26 +1.396 [+0.912, +1.890] 22:3 livEng-ace|pooled n=26 +1.399 [+0.848, +1.940] 21:5 livEng-aceseed|pooled n=26 +0.518 [+0.088, +1.021] 16:9 livP1-stock|pooled n=26 +1.001 [+0.555, +1.468] 16:5 livP1-aceseed|pooled n=26 -0.107 [-0.319, +0.114] 7:11 livP1-bare|pooled n=26 +0.865 [+0.373, +1.372] 14:9 livKF-livEng|pooled n=26 +0.314 [-0.024, +0.694] 14:8 livKF-livP1|pooled n=26 +0.940 [+0.478, +1.467] 15:5 livEng-livP1|pooled n=26 +0.625 [+0.213, +1.090] 16:7 aceseed-ace|pooled n=26 +0.880 [+0.318, +1.451] 12:8 aceKF-ace|pooled n=26 +1.337 [+0.766, +1.906] 17:6 aceKF-aceseed|pooled n=26 +0.457 [+0.096, +0.889] 12:8 ace-bare|pooled n=26 +0.091 [-0.102, +0.319] 13:8 prior-bare|pooled n=26 +0.094 [-0.010, +0.218] 13:6 stock-bare|pooled n=26 -0.137 [-0.234, -0.048] 6:13 stock-ace|pooled n=26 -0.228 [-0.429, -0.072] 2:18 aceseed-bare|pooled n=26 +0.972 [+0.416, +1.552] 15:8 aceKF-bare|pooled n=26 +1.429 [+0.914, +1.956] 20:4 livE5-stock|pooled n= 6 +0.118 [+0.005, +0.290] 2:0 livE5-bare|pooled n= 6 -0.051 [-0.295, +0.155] 2:2 livE5-prior|pooled n= 6 +0.062 [-0.273, +0.335] 3:1 livE5-ace|pooled n= 6 +0.039 [-0.295, +0.388] 2:3 livKF-bare|dev n=10 +2.158 [+1.309, +2.988] 8:1 livKF-prior|dev n=10 +1.959 [+1.039, +2.856] 7:2 livKF-ace|dev n=10 +1.771 [+0.769, +2.743] 8:2 livKF-aceseed|dev n=10 +0.576 [-0.168, +1.474] 5:3 livKF-stock|dev n=10 +2.194 [+1.405, +2.981] 9:1 livKF-aceKF|dev n=10 +0.617 [-0.006, +1.438] 5:3 livEng-stock|dev n=10 +1.875 [+1.136, +2.610] 8:2 livEng-bare|dev n=10 +1.839 [+1.065, +2.611] 7:1 livEng-prior|dev n=10 +1.640 [+0.728, +2.520] 7:2 livEng-ace|dev n=10 +1.452 [+0.407, +2.410] 8:2 livEng-aceseed|dev n=10 +0.257 [-0.309, +0.836] 5:4 livP1-stock|dev n=10 +1.792 [+1.075, +2.498] 9:0 livP1-aceseed|dev n=10 +0.174 [-0.218, +0.584] 5:2 livP1-bare|dev n=10 +1.756 [+0.994, +2.507] 7:2 livKF-livEng|dev n=10 +0.319 [-0.195, +0.898] 5:3 livKF-livP1|dev n=10 +0.402 [-0.130, +1.307] 3:3 livEng-livP1|dev n=10 +0.083 [-0.339, +0.526] 5:4 aceseed-ace|dev n=10 +1.195 [+0.078, +2.281] 6:3 aceKF-ace|dev n=10 +1.154 [+0.060, +2.186] 5:4 aceKF-aceseed|dev n=10 -0.041 [-0.293, +0.227] 3:4 ace-bare|dev n=10 +0.386 [+0.014, +0.909] 7:1 prior-bare|dev n=10 +0.199 [-0.006, +0.482] 4:2 stock-bare|dev n=10 -0.037 [-0.141, +0.059] 3:3 stock-ace|dev n=10 -0.423 [-0.926, -0.113] 1:8 aceseed-bare|dev n=10 +1.581 [+0.543, +2.540] 7:2 aceKF-bare|dev n=10 +1.540 [+0.569, +2.484] 7:2 livE5-stock|dev n= 1 +0.576 [+0.576, +0.576] 1:0 livE5-bare|dev n= 1 +0.320 [+0.320, +0.320] 1:0 livE5-prior|dev n= 1 +0.489 [+0.489, +0.489] 1:0 livE5-ace|dev n= 1 +0.323 [+0.323, +0.323] 1:0 livKF-bare|holdout n=16 +1.584 [+1.056, +2.151] 14:2 livKF-prior|holdout n=16 +1.555 [+1.026, +2.139] 15:1 livKF-ace|holdout n=16 +1.677 [+1.101, +2.314] 14:2 livKF-aceseed|holdout n=16 +0.993 [+0.336, +1.711] 12:4 livKF-stock|holdout n=16 +1.783 [+1.227, +2.406] 15:0 livKF-aceKF|holdout n=16 +0.225 [-0.263, +0.783] 8:8 livEng-stock|holdout n=16 +1.471 [+0.903, +2.094] 14:1 livEng-bare|holdout n=16 +1.272 [+0.742, +1.840] 14:2 livEng-prior|holdout n=16 +1.244 [+0.710, +1.827] 15:1 livEng-ace|holdout n=16 +1.365 [+0.761, +2.016] 13:3 livEng-aceseed|holdout n=16 +0.682 [+0.063, +1.388] 11:5 livP1-stock|holdout n=16 +0.507 [+0.052, +1.012] 7:5 livP1-aceseed|holdout n=16 -0.283 [-0.497, -0.076] 2:9 livP1-bare|holdout n=16 +0.308 [-0.194, +0.872] 7:7 livKF-livEng|holdout n=16 +0.312 [-0.116, +0.825] 9:5 livKF-livP1|holdout n=16 +1.276 [+0.690, +1.948] 12:2 livEng-livP1|holdout n=16 +0.964 [+0.400, +1.623] 11:3 aceseed-ace|holdout n=16 +0.684 [+0.137, +1.318] 6:5 aceKF-ace|holdout n=16 +1.452 [+0.836, +2.094] 12:2 aceKF-aceseed|holdout n=16 +0.768 [+0.228, +1.391] 9:4 ace-bare|holdout n=16 -0.093 [-0.262, +0.066] 6:7 prior-bare|holdout n=16 +0.028 [-0.067, +0.114] 9:4 stock-bare|holdout n=16 -0.199 [-0.335, -0.077] 3:10 stock-ace|holdout n=16 -0.106 [-0.221, +0.021] 1:10 aceseed-bare|holdout n=16 +0.591 [+0.004, +1.255] 8:6 aceKF-bare|holdout n=16 +1.359 [+0.770, +1.972] 13:2 livE5-stock|holdout n= 5 +0.027 [-0.003, +0.063] 1:0 livE5-bare|holdout n= 5 -0.126 [-0.361, +0.102] 1:2 livE5-prior|holdout n= 5 -0.024 [-0.366, +0.264] 2:1 livE5-ace|holdout n= 5 -0.018 [-0.378, +0.366] 1:3 livKF-bare|p1_applicable n=14 +2.160 [+1.619, +2.676] 13:1 livKF-prior|p1_applicable n=14 +1.984 [+1.377, +2.558] 12:2 livKF-ace|p1_applicable n=14 +1.876 [+1.197, +2.485] 12:2 livKF-aceseed|p1_applicable n=14 +0.098 [-0.329, +0.588] 7:5 livKF-stock|p1_applicable n=14 +2.159 [+1.626, +2.660] 13:0 livKF-aceKF|p1_applicable n=14 +0.127 [-0.337, +0.738] 4:8 livEng-stock|p1_applicable n=14 +1.852 [+1.193, +2.502] 11:3 livEng-bare|p1_applicable n=14 +1.853 [+1.202, +2.509] 11:2 livEng-prior|p1_applicable n=14 +1.677 [+0.924, +2.398] 11:3 livEng-ace|p1_applicable n=14 +1.569 [+0.714, +2.349] 11:3 livEng-aceseed|p1_applicable n=14 -0.209 [-0.603, +0.210] 5:8 livP1-stock|p1_applicable n=14 +1.927 [+1.362, +2.488] 12:1 livP1-aceseed|p1_applicable n=14 -0.134 [-0.408, +0.136] 6:6 livP1-bare|p1_applicable n=14 +1.929 [+1.350, +2.505] 13:1 livKF-livEng|p1_applicable n=14 +0.307 [-0.164, +0.905] 7:5 livKF-livP1|p1_applicable n=14 +0.231 [-0.024, +0.611] 5:3 livEng-livP1|p1_applicable n=14 -0.075 [-0.409, +0.268] 5:6 aceseed-ace|p1_applicable n=14 +1.778 [+0.911, +2.593] 11:3 aceKF-ace|p1_applicable n=14 +1.748 [+0.888, +2.543] 11:3 aceKF-aceseed|p1_applicable n=14 -0.030 [-0.306, +0.261] 5:7 ace-bare|p1_applicable n=14 +0.284 [+0.009, +0.668] 10:3 prior-bare|p1_applicable n=14 +0.177 [+0.025, +0.383] 8:3 stock-bare|p1_applicable n=14 +0.001 [-0.058, +0.064] 5:4 stock-ace|p1_applicable n=14 -0.283 [-0.661, -0.009] 1:11 aceseed-bare|p1_applicable n=14 +2.062 [+1.335, +2.770] 12:2 aceKF-bare|p1_applicable n=14 +2.033 [+1.354, +2.702] 13:1 livE5-stock|p1_applicable n= 2 +0.054 [-0.003, +0.112] 1:0 livE5-bare|p1_applicable n= 2 +0.142 [+0.011, +0.273] 1:0 livE5-prior|p1_applicable n= 2 +0.215 [-0.002, +0.433] 1:0 livE5-ace|p1_applicable n= 2 +0.369 [-0.183, +0.920] 1:1 livKF-bare|holdout_p1 n= 7 +1.821 [+1.041, +2.585] 6:1 livKF-prior|holdout_p1 n= 7 +1.774 [+1.002, +2.547] 6:1 livKF-ace|holdout_p1 n= 7 +1.876 [+1.162, +2.555] 6:1 livKF-aceseed|holdout_p1 n= 7 +0.271 [-0.511, +1.160] 4:3 livKF-stock|holdout_p1 n= 7 +1.864 [+1.098, +2.611] 6:0 livKF-aceKF|holdout_p1 n= 7 +0.205 [-0.642, +1.443] 2:5 livEng-stock|holdout_p1 n= 7 +1.504 [+0.615, +2.303] 6:1 livEng-bare|holdout_p1 n= 7 +1.461 [+0.605, +2.239] 6:1 livEng-prior|holdout_p1 n= 7 +1.414 [+0.583, +2.205] 6:1 livEng-ace|holdout_p1 n= 7 +1.516 [+0.660, +2.272] 6:1 livEng-aceseed|holdout_p1 n= 7 -0.089 [-0.742, +0.620] 3:4 livP1-stock|holdout_p1 n= 7 +1.357 [+0.559, +2.160] 5:1 livP1-aceseed|holdout_p1 n= 7 -0.236 [-0.620, +0.146] 2:4 livP1-bare|holdout_p1 n= 7 +1.314 [+0.519, +2.147] 6:1 livKF-livEng|holdout_p1 n= 7 +0.360 [-0.491, +1.538] 3:3 livKF-livP1|holdout_p1 n= 7 +0.507 [+0.056, +1.227] 4:1 livEng-livP1|holdout_p1 n= 7 +0.147 [-0.373, +0.678] 3:2 aceseed-ace|holdout_p1 n= 7 +1.605 [+0.514, +2.679] 5:2 aceKF-ace|holdout_p1 n= 7 +1.671 [+0.703, +2.620] 6:1 aceKF-aceseed|holdout_p1 n= 7 +0.066 [-0.428, +0.580] 3:4 ace-bare|holdout_p1 n= 7 -0.055 [-0.233, +0.078] 3:3 prior-bare|holdout_p1 n= 7 +0.047 [-0.040, +0.134] 4:2 stock-bare|holdout_p1 n= 7 -0.043 [-0.126, +0.040] 2:3 stock-ace|holdout_p1 n= 7 +0.011 [-0.156, +0.258] 1:4 aceseed-bare|holdout_p1 n= 7 +1.550 [+0.400, +2.653] 5:2 aceKF-bare|holdout_p1 n= 7 +1.616 [+0.639, +2.589] 6:1 livE5-stock|holdout_p1 n= 2 +0.054 [-0.003, +0.112] 1:0 livE5-bare|holdout_p1 n= 2 +0.142 [+0.011, +0.273] 1:0 livE5-prior|holdout_p1 n= 2 +0.215 [-0.002, +0.433] 1:0 livE5-ace|holdout_p1 n= 2 +0.369 [-0.183, +0.920] 1:1 livKF-bare|holdout_nonp1 n= 9 +1.399 [+0.760, +2.187] 8:1 livKF-prior|holdout_nonp1 n= 9 +1.385 [+0.748, +2.246] 9:0 livKF-ace|holdout_nonp1 n= 9 +1.522 [+0.713, +2.513] 8:1 livKF-aceseed|holdout_nonp1 n= 9 +1.555 [+0.737, +2.550] 8:1 livKF-stock|holdout_nonp1 n= 9 +1.720 [+0.986, +2.651] 9:0 livKF-aceKF|holdout_nonp1 n= 9 +0.240 [-0.213, +0.686] 6:3 livEng-stock|holdout_nonp1 n= 9 +1.446 [+0.718, +2.368] 8:0 livEng-bare|holdout_nonp1 n= 9 +1.125 [+0.490, +1.908] 8:1 livEng-prior|holdout_nonp1 n= 9 +1.111 [+0.453, +1.937] 9:0 livEng-ace|holdout_nonp1 n= 9 +1.248 [+0.441, +2.234] 7:2 livEng-aceseed|holdout_nonp1 n= 9 +1.281 [+0.460, +2.277] 8:1 livP1-stock|holdout_nonp1 n= 9 -0.154 [-0.371, +0.045] 2:4 livP1-aceseed|holdout_nonp1 n= 9 -0.319 [-0.563, -0.112] 0:5 livP1-bare|holdout_nonp1 n= 9 -0.474 [-0.755, -0.216] 1:6 livKF-livEng|holdout_nonp1 n= 9 +0.274 [+0.034, +0.515] 6:2 livKF-livP1|holdout_nonp1 n= 9 +1.874 [+1.067, +2.833] 8:1 livEng-livP1|holdout_nonp1 n= 9 +1.600 [+0.833, +2.533] 8:1 aceseed-ace|holdout_nonp1 n= 9 -0.032 [-0.186, +0.131] 1:3 aceKF-ace|holdout_nonp1 n= 9 +1.282 [+0.545, +2.142] 6:1 aceKF-aceseed|holdout_nonp1 n= 9 +1.314 [+0.535, +2.201] 6:0 ace-bare|holdout_nonp1 n= 9 -0.123 [-0.395, +0.133] 3:4 prior-bare|holdout_nonp1 n= 9 +0.014 [-0.141, +0.150] 5:2 stock-bare|holdout_nonp1 n= 9 -0.320 [-0.523, -0.127] 1:7 stock-ace|holdout_nonp1 n= 9 -0.197 [-0.332, -0.085] 0:6 aceseed-bare|holdout_nonp1 n= 9 -0.155 [-0.386, +0.056] 3:4 aceKF-bare|holdout_nonp1 n= 9 +1.159 [+0.465, +1.914] 7:1 livE5-stock|holdout_nonp1 n= 3 +0.008 [-0.007, +0.024] 0:0 livE5-bare|holdout_nonp1 n= 3 -0.304 [-0.504, -0.105] 0:2 livE5-prior|holdout_nonp1 n= 3 -0.183 [-0.567, +0.200] 1:1 livE5-ace|holdout_nonp1 n= 3 -0.275 [-0.532, -0.018] 0:2 livKF-bare|engine_card n=25 +1.827 [+1.337, +2.322] 21:3 livKF-prior|engine_card n=25 +1.791 [+1.308, +2.289] 22:2 livKF-ace|engine_card n=25 +1.845 [+1.349, +2.364] 22:3 livKF-aceseed|engine_card n=25 +0.835 [+0.317, +1.416] 16:7 livKF-stock|engine_card n=25 +1.972 [+1.489, +2.474] 23:1 livKF-aceKF|engine_card n=25 +0.353 [-0.059, +0.814] 12:11 livEng-stock|engine_card n=25 +1.700 [+1.239, +2.190] 22:2 livEng-bare|engine_card n=25 +1.554 [+1.096, +2.026] 21:2 livEng-prior|engine_card n=25 +1.519 [+1.056, +2.003] 22:2 livEng-ace|engine_card n=25 +1.573 [+1.092, +2.076] 21:4 livEng-aceseed|engine_card n=25 +0.562 [+0.111, +1.065] 16:8 livP1-stock|engine_card n=25 +1.004 [+0.553, +1.486] 15:5 livP1-aceseed|engine_card n=25 -0.133 [-0.342, +0.089] 6:11 livP1-bare|engine_card n=25 +0.859 [+0.358, +1.384] 13:9 livKF-livEng|engine_card n=25 +0.273 [-0.062, +0.652] 13:8 livKF-livP1|engine_card n=25 +0.968 [+0.487, +1.509] 14:5 livEng-livP1|engine_card n=25 +0.695 [+0.279, +1.166] 16:6 aceseed-ace|engine_card n=25 +1.011 [+0.482, +1.570] 12:7 aceKF-ace|engine_card n=25 +1.492 [+0.955, +2.032] 17:5 aceKF-aceseed|engine_card n=25 +0.482 [+0.107, +0.911] 12:7 ace-bare|engine_card n=25 -0.019 [-0.146, +0.101] 12:8 prior-bare|engine_card n=25 +0.035 [-0.035, +0.099] 12:6 stock-bare|engine_card n=25 -0.146 [-0.246, -0.052] 5:13 stock-ace|engine_card n=25 -0.127 [-0.207, -0.041] 2:17 aceseed-bare|engine_card n=25 +0.992 [+0.418, +1.593] 14:8 aceKF-bare|engine_card n=25 +1.473 [+0.932, +2.013] 19:4 livE5-stock|engine_card n= 6 +0.118 [+0.005, +0.290] 2:0 livE5-bare|engine_card n= 6 -0.051 [-0.295, +0.155] 2:2 livE5-prior|engine_card n= 6 +0.062 [-0.273, +0.335] 3:1 livE5-ace|engine_card n= 6 +0.039 [-0.295, +0.388] 2:3 livKF-bare|no_p1_engine_card n=12 +1.390 [+0.674, +2.192] 9:2 livKF-prior|no_p1_engine_card n=12 +1.392 [+0.671, +2.215] 10:1 livKF-ace|no_p1_engine_card n=12 +1.524 [+0.724, +2.421] 10:2 livKF-aceseed|no_p1_engine_card n=12 +1.690 [+0.899, +2.575] 10:2 livKF-stock|no_p1_engine_card n=12 +1.687 [+0.934, +2.540] 11:1 livKF-aceKF|no_p1_engine_card n=12 +0.665 [+0.074, +1.382] 9:3 livEng-stock|no_p1_engine_card n=12 +1.364 [+0.791, +2.097] 11:0 livEng-bare|no_p1_engine_card n=12 +1.066 [+0.559, +1.690] 10:1 livEng-prior|no_p1_engine_card n=12 +1.069 [+0.547, +1.733] 11:0 livEng-ace|no_p1_engine_card n=12 +1.200 [+0.581, +1.966] 10:2 livEng-aceseed|no_p1_engine_card n=12 +1.367 [+0.733, +2.128] 11:1 livP1-stock|no_p1_engine_card n=12 -0.079 [-0.260, +0.086] 4:4 livP1-aceseed|no_p1_engine_card n=12 -0.076 [-0.382, +0.282] 1:5 livP1-bare|no_p1_engine_card n=12 -0.377 [-0.600, -0.165] 1:8 livKF-livEng|no_p1_engine_card n=12 +0.323 [-0.101, +0.805] 7:3 livKF-livP1|no_p1_engine_card n=12 +1.766 [+0.962, +2.657] 10:2 livEng-livP1|no_p1_engine_card n=12 +1.443 [+0.834, +2.183] 11:1 aceseed-ace|no_p1_engine_card n=12 -0.167 [-0.422, +0.042] 1:5 aceKF-ace|no_p1_engine_card n=12 +0.858 [+0.218, +1.620] 6:3 aceKF-aceseed|no_p1_engine_card n=12 +1.025 [+0.393, +1.765] 7:1 ace-bare|no_p1_engine_card n=12 -0.134 [-0.342, +0.068] 3:5 prior-bare|no_p1_engine_card n=12 -0.002 [-0.122, +0.103] 5:3 stock-bare|no_p1_engine_card n=12 -0.298 [-0.455, -0.141] 1:9 stock-ace|no_p1_engine_card n=12 -0.164 [-0.271, -0.070] 1:7 aceseed-bare|no_p1_engine_card n=12 -0.301 [-0.631, -0.024] 3:6 aceKF-bare|no_p1_engine_card n=12 +0.724 [+0.089, +1.410] 7:3 livE5-stock|no_p1_engine_card n= 4 +0.150 [-0.002, +0.428] 1:0 livE5-bare|no_p1_engine_card n= 4 -0.148 [-0.453, +0.158] 1:2 livE5-prior|no_p1_engine_card n= 4 -0.015 [-0.518, +0.369] 2:1 livE5-ace|no_p1_engine_card n= 4 -0.126 [-0.477, +0.175] 1:2 livKF-bare|holdout_ace_named n= 7 +1.496 [+0.791, +2.242] 6:1 livKF-prior|holdout_ace_named n= 7 +1.446 [+0.726, +2.251] 6:1 livKF-ace|holdout_ace_named n= 7 +1.570 [+0.911, +2.263] 6:1 livKF-aceseed|holdout_ace_named n= 7 +0.949 [+0.334, +1.632] 6:1 livKF-stock|holdout_ace_named n= 7 +1.589 [+0.903, +2.308] 6:0 livKF-aceKF|holdout_ace_named n= 7 +0.297 [-0.616, +1.473] 3:4 livEng-stock|holdout_ace_named n= 7 +0.991 [+0.243, +1.771] 6:1 livEng-bare|holdout_ace_named n= 7 +0.898 [+0.169, +1.657] 6:1 livEng-prior|holdout_ace_named n= 7 +0.848 [+0.104, +1.655] 6:1 livEng-ace|holdout_ace_named n= 7 +0.972 [+0.181, +1.766] 5:2 livEng-aceseed|holdout_ace_named n= 7 +0.351 [-0.196, +0.878] 5:2 livP1-stock|holdout_ace_named n= 7 +0.418 [-0.311, +1.296] 3:3 livP1-aceseed|holdout_ace_named n= 7 -0.222 [-0.611, +0.148] 2:4 livP1-bare|holdout_ace_named n= 7 +0.325 [-0.464, +1.251] 4:3 livKF-livEng|holdout_ace_named n= 7 +0.598 [-0.320, +1.678] 4:3 livKF-livP1|holdout_ace_named n= 7 +1.171 [+0.487, +1.906] 6:1 livEng-livP1|holdout_ace_named n= 7 +0.572 [-0.096, +1.196] 5:1 aceseed-ace|holdout_ace_named n= 7 +0.622 [-0.078, +1.536] 4:3 aceKF-ace|holdout_ace_named n= 7 +1.273 [+0.448, +2.069] 6:1 aceKF-aceseed|holdout_ace_named n= 7 +0.652 [-0.045, +1.358] 5:2 ace-bare|holdout_ace_named n= 7 -0.074 [-0.345, +0.182] 4:3 prior-bare|holdout_ace_named n= 7 +0.050 [-0.031, +0.127] 4:2 stock-bare|holdout_ace_named n= 7 -0.093 [-0.298, +0.083] 3:3 stock-ace|holdout_ace_named n= 7 -0.019 [-0.196, +0.234] 1:4 aceseed-bare|holdout_ace_named n= 7 +0.547 [-0.167, +1.511] 4:3 aceKF-bare|holdout_ace_named n= 7 +1.199 [+0.320, +2.070] 6:1 livE5-stock|holdout_ace_named n= 2 +0.054 [-0.003, +0.112] 1:0 livE5-bare|holdout_ace_named n= 2 +0.142 [+0.011, +0.273] 1:0 livE5-prior|holdout_ace_named n= 2 +0.215 [-0.002, +0.433] 1:0 livE5-ace|holdout_ace_named n= 2 +0.369 [-0.183, +0.920] 1:1 livKF-bare|holdout_ace_unnamed n= 9 +1.652 [+0.942, +2.463] 8:1 livKF-prior|holdout_ace_unnamed n= 9 +1.640 [+0.930, +2.486] 9:0 livKF-ace|holdout_ace_unnamed n= 9 +1.760 [+0.898, +2.766] 8:1 livKF-aceseed|holdout_ace_unnamed n= 9 +1.028 [+0.024, +2.215] 6:3 livKF-stock|holdout_ace_unnamed n= 9 +1.934 [+1.158, +2.872] 9:0 livKF-aceKF|holdout_ace_unnamed n= 9 +0.169 [-0.226, +0.602] 5:4 livEng-stock|holdout_ace_unnamed n= 9 +1.845 [+1.094, +2.752] 8:0 livEng-bare|holdout_ace_unnamed n= 9 +1.563 [+0.871, +2.345] 8:1 livEng-prior|holdout_ace_unnamed n= 9 +1.552 [+0.865, +2.366] 9:0 livEng-ace|holdout_ace_unnamed n= 9 +1.671 [+0.836, +2.651] 8:1 livEng-aceseed|holdout_ace_unnamed n= 9 +0.939 [-0.040, +2.087] 6:3 livP1-stock|holdout_ace_unnamed n= 9 +0.576 [+0.021, +1.174] 4:2 livP1-aceseed|holdout_ace_unnamed n= 9 -0.330 [-0.564, -0.118] 0:5 livP1-bare|holdout_ace_unnamed n= 9 +0.294 [-0.333, +0.983] 3:4 livKF-livEng|holdout_ace_unnamed n= 9 +0.089 [-0.073, +0.245] 5:2 livKF-livP1|holdout_ace_unnamed n= 9 +1.358 [+0.503, +2.433] 6:1 livEng-livP1|holdout_ace_unnamed n= 9 +1.269 [+0.433, +2.310] 6:2 aceseed-ace|holdout_ace_unnamed n= 9 +0.732 [-0.059, +1.578] 2:2 aceKF-ace|holdout_ace_unnamed n= 9 +1.591 [+0.707, +2.537] 6:1 aceKF-aceseed|holdout_ace_unnamed n= 9 +0.859 [+0.095, +1.817] 4:2 ace-bare|holdout_ace_unnamed n= 9 -0.108 [-0.332, +0.083] 2:4 prior-bare|holdout_ace_unnamed n= 9 +0.012 [-0.147, +0.152] 5:2 stock-bare|holdout_ace_unnamed n= 9 -0.282 [-0.460, -0.130] 0:7 stock-ace|holdout_ace_unnamed n= 9 -0.174 [-0.310, -0.067] 0:6 aceseed-bare|holdout_ace_unnamed n= 9 +0.624 [-0.206, +1.526] 4:3 aceKF-bare|holdout_ace_unnamed n= 9 +1.483 [+0.674, +2.332] 7:1 livE5-stock|holdout_ace_unnamed n= 3 +0.008 [-0.007, +0.024] 0:0 livE5-bare|holdout_ace_unnamed n= 3 -0.304 [-0.504, -0.105] 0:2 livE5-prior|holdout_ace_unnamed n= 3 -0.183 [-0.567, +0.200] 1:1 livE5-ace|holdout_ace_unnamed n= 3 -0.275 [-0.532, -0.018] 0:2 livKF-bare|e5_problems n= 6 +0.823 [+0.090, +1.681] 4:2 livKF-prior|e5_problems n= 6 +0.936 [+0.306, +1.742] 5:1 livKF-ace|e5_problems n= 6 +0.913 [+0.097, +1.753] 4:2 livKF-aceseed|e5_problems n= 6 +0.932 [+0.149, +1.768] 4:2 livKF-stock|e5_problems n= 6 +0.992 [+0.268, +1.812] 5:1 livKF-aceKF|e5_problems n= 6 +0.977 [+0.019, +2.134] 4:2 livEng-stock|e5_problems n= 6 +0.438 [-0.110, +0.934] 4:1 livEng-bare|e5_problems n= 6 +0.269 [-0.224, +0.712] 4:2 livEng-prior|e5_problems n= 6 +0.382 [-0.137, +0.779] 5:1 livEng-ace|e5_problems n= 6 +0.359 [-0.322, +0.996] 4:2 livEng-aceseed|e5_problems n= 6 +0.378 [-0.287, +0.975] 4:2 livP1-stock|e5_problems n= 6 +0.145 [+0.029, +0.274] 3:0 livP1-aceseed|e5_problems n= 6 +0.085 [-0.119, +0.315] 2:2 livP1-bare|e5_problems n= 6 -0.024 [-0.236, +0.166] 2:2 livKF-livEng|e5_problems n= 6 +0.554 [-0.419, +1.882] 3:2 livKF-livP1|e5_problems n= 6 +0.847 [+0.125, +1.628] 4:2 livEng-livP1|e5_problems n= 6 +0.293 [-0.355, +0.868] 4:2 aceseed-ace|e5_problems n= 6 -0.019 [-0.065, +0.029] 1:3 aceKF-ace|e5_problems n= 6 -0.064 [-0.554, +0.439] 2:3 aceKF-aceseed|e5_problems n= 6 -0.044 [-0.498, +0.411] 2:2 ace-bare|e5_problems n= 6 -0.090 [-0.348, +0.164] 2:2 prior-bare|e5_problems n= 6 -0.113 [-0.288, +0.056] 1:3 stock-bare|e5_problems n= 6 -0.170 [-0.339, -0.002] 1:3 stock-ace|e5_problems n= 6 -0.079 [-0.375, +0.235] 1:4 aceseed-bare|e5_problems n= 6 -0.110 [-0.328, +0.108] 2:3 aceKF-bare|e5_problems n= 6 -0.154 [-0.552, +0.243] 3:3 livE5-stock|e5_problems n= 6 +0.118 [+0.005, +0.290] 2:0 livE5-bare|e5_problems n= 6 -0.051 [-0.295, +0.155] 2:2 livE5-prior|e5_problems n= 6 +0.062 [-0.273, +0.335] 3:1 livE5-ace|e5_problems n= 6 +0.039 [-0.295, +0.388] 2:3 SECONDARY { "did_seed_living_vs_ace|p1": { "n": 14, "mean_diff": 0.149, "ci90": [ -0.27, 0.655 ], "wins": 8, "losses": 6 }, "interaction_livKF-livP1-livEng+stock|pooled": { "n": 26, "mean_diff": -0.687, "ci90": [ -1.297, -0.057 ], "wins": 9, "losses": 15 }, "interaction_livKF-livP1-livEng+stock|dev": { "n": 10, "mean_diff": -1.473, "ci90": [ -2.459, -0.398 ], "wins": 2, "losses": 8 }, "interaction_livKF-livP1-livEng+stock|holdout": { "n": 16, "mean_diff": -0.196, "ci90": [ -0.894, 0.508 ], "wins": 7, "losses": 7 }, "interaction_livKF-livP1-livEng+stock|p1_applicable": { "n": 14, "mean_diff": -1.621, "ci90": [ -2.449, -0.71 ], "wins": 2, "losses": 11 }, "interaction_livKF-livP1-livEng+stock|engine_card": { "n": 25, "mean_diff": -0.731, "ci90": [ -1.368, -0.091 ], "wins": 8, "losses": 15 }, "did_library_living_vs_ace(livKF-stock)-(aceKF-ace)|pooled": { "n": 26, "mean_diff": 0.604, "ci90": [ 0.158, 1.102 ], "wins": 18, "losses": 8 }, "did_library_living_vs_ace(livKF-stock)-(aceKF-ace)|dev": { "n": 10, "mean_diff": 1.04, "ci90": [ 0.255, 1.927 ], "wins": 8, "losses": 2 }, "did_library_living_vs_ace(livKF-stock)-(aceKF-ace)|holdout": { "n": 16, "mean_diff": 0.331, "ci90": [ -0.155, 0.899 ], "wins": 10, "losses": 6 }, "did_library_living_vs_ace(livKF-stock)-(aceKF-ace)|p1_applicable": { "n": 14, "mean_diff": 0.411, "ci90": [ -0.218, 1.172 ], "wins": 7, "losses": 7 }, "leave_one_out_livKF-ace|pooled": { "min": 1.551, "max": 1.845 }, "reweighted_to_pool_prevalence_7of28_livKF-ace": 1.612, "leave_one_out_livKF-aceseed|pooled": { "min": 0.636, "max": 0.912 }, "reweighted_to_pool_prevalence_7of28_livKF-aceseed": 1.292, "leave_one_out_livKF-bare|pooled": { "min": 1.685, "max": 1.894 }, "reweighted_to_pool_prevalence_7of28_livKF-bare": 1.582, "leave_one_out_livKF-prior|pooled": { "min": 1.576, "max": 1.792 }, "reweighted_to_pool_prevalence_7of28_livKF-prior": 1.54, "leave_one_out_livKF-stock|pooled": { "min": 1.788, "max": 2.026 }, "reweighted_to_pool_prevalence_7of28_livKF-stock": 1.805 } ANY-REPEAT DIFFERENCES {} DECISIONS { "C1_system_claim_livKF_over_ace_and_aceseed": true, "C2_durability_engine_only_over_stock_bare_prior_ace": true, "C3_boolean_aceseed_gain_ge_livP1_gain_minus_0.25": true, "C3_values": { "livP1-stock|p1": { "n": 14, "mean_diff": 1.927, "ci90": [ 1.362, 2.488 ], "wins": 12, "losses": 1 }, "aceseed-ace|p1": { "n": 14, "mean_diff": 1.778, "ci90": [ 0.911, 2.593 ], "wins": 11, "losses": 3 } }, "C1_holdout_only": true, "C2_holdout_only": true }