| comparison (subset) | n problems | mean diff (x, capped) | 90% CI | W:L |
|---|---|---|---|---|
| livKF-bare|pooled | 26 | +1.805 | [+1.330, +2.280] | 22:3 |
| livKF-prior|pooled | 26 | +1.711 | [+1.225, +2.206] | 22:3 |
| livKF-ace|pooled | 26 | +1.713 | [+1.187, +2.248] | 22:4 |
| livKF-aceseed|pooled | 26 | +0.833 | [+0.327, +1.399] | 17:7 |
| livKF-stock|pooled | 26 | +1.941 | [+1.471, +2.422] | 24:1 |
| livKF-aceKF|pooled | 26 | +0.376 | [-0.028, +0.843] | 13:11 |
| livEng-stock|pooled | 26 | +1.627 | [+1.162, +2.104] | 22:3 |
| livEng-bare|pooled | 26 | +1.490 | [+1.041, +1.952] | 21:3 |
| livEng-prior|pooled | 26 | +1.396 | [+0.912, +1.890] | 22:3 |
| livEng-ace|pooled | 26 | +1.399 | [+0.848, +1.940] | 21:5 |
| livEng-aceseed|pooled | 26 | +0.518 | [+0.088, +1.021] | 16:9 |
| livP1-stock|pooled | 26 | +1.001 | [+0.555, +1.468] | 16:5 |
| livP1-aceseed|pooled | 26 | -0.107 | [-0.319, +0.114] | 7:11 |
| livP1-bare|pooled | 26 | +0.865 | [+0.373, +1.372] | 14:9 |
| livKF-livEng|pooled | 26 | +0.314 | [-0.024, +0.694] | 14:8 |
| livKF-livP1|pooled | 26 | +0.940 | [+0.478, +1.467] | 15:5 |
| livEng-livP1|pooled | 26 | +0.625 | [+0.213, +1.090] | 16:7 |
| aceseed-ace|pooled | 26 | +0.880 | [+0.318, +1.451] | 12:8 |
| aceKF-ace|pooled | 26 | +1.337 | [+0.766, +1.906] | 17:6 |
| aceKF-aceseed|pooled | 26 | +0.457 | [+0.096, +0.889] | 12:8 |
| ace-bare|pooled | 26 | +0.091 | [-0.102, +0.319] | 13:8 |
| prior-bare|pooled | 26 | +0.094 | [-0.010, +0.218] | 13:6 |
| stock-bare|pooled | 26 | -0.137 | [-0.234, -0.048] | 6:13 |
| stock-ace|pooled | 26 | -0.228 | [-0.429, -0.072] | 2:18 |
| aceseed-bare|pooled | 26 | +0.972 | [+0.416, +1.552] | 15:8 |
| aceKF-bare|pooled | 26 | +1.429 | [+0.914, +1.956] | 20:4 |
| livE5-stock|pooled | 6 | +0.118 | [+0.005, +0.290] | 2:0 |
| livE5-bare|pooled | 6 | -0.051 | [-0.295, +0.155] | 2:2 |
| livE5-prior|pooled | 6 | +0.062 | [-0.273, +0.335] | 3:1 |
| livE5-ace|pooled | 6 | +0.039 | [-0.295, +0.388] | 2:3 |
| livKF-bare|dev | 10 | +2.158 | [+1.309, +2.988] | 8:1 |
| livKF-prior|dev | 10 | +1.959 | [+1.039, +2.856] | 7:2 |
| livKF-ace|dev | 10 | +1.771 | [+0.769, +2.743] | 8:2 |
| livKF-aceseed|dev | 10 | +0.576 | [-0.168, +1.474] | 5:3 |
| livKF-stock|dev | 10 | +2.194 | [+1.405, +2.981] | 9:1 |
| livKF-aceKF|dev | 10 | +0.617 | [-0.006, +1.438] | 5:3 |
| livEng-stock|dev | 10 | +1.875 | [+1.136, +2.610] | 8:2 |
| livEng-bare|dev | 10 | +1.839 | [+1.065, +2.611] | 7:1 |
| livEng-prior|dev | 10 | +1.640 | [+0.728, +2.520] | 7:2 |
| livEng-ace|dev | 10 | +1.452 | [+0.407, +2.410] | 8:2 |
| livEng-aceseed|dev | 10 | +0.257 | [-0.309, +0.836] | 5:4 |
| livP1-stock|dev | 10 | +1.792 | [+1.075, +2.498] | 9:0 |
| livP1-aceseed|dev | 10 | +0.174 | [-0.218, +0.584] | 5:2 |
| livP1-bare|dev | 10 | +1.756 | [+0.994, +2.507] | 7:2 |
| livKF-livEng|dev | 10 | +0.319 | [-0.195, +0.898] | 5:3 |
| livKF-livP1|dev | 10 | +0.402 | [-0.130, +1.307] | 3:3 |
| livEng-livP1|dev | 10 | +0.083 | [-0.339, +0.526] | 5:4 |
| aceseed-ace|dev | 10 | +1.195 | [+0.078, +2.281] | 6:3 |
| aceKF-ace|dev | 10 | +1.154 | [+0.060, +2.186] | 5:4 |
| aceKF-aceseed|dev | 10 | -0.041 | [-0.293, +0.227] | 3:4 |
| ace-bare|dev | 10 | +0.386 | [+0.014, +0.909] | 7:1 |
| prior-bare|dev | 10 | +0.199 | [-0.006, +0.482] | 4:2 |
| stock-bare|dev | 10 | -0.037 | [-0.141, +0.059] | 3:3 |
| stock-ace|dev | 10 | -0.423 | [-0.926, -0.113] | 1:8 |
| aceseed-bare|dev | 10 | +1.581 | [+0.543, +2.540] | 7:2 |
| aceKF-bare|dev | 10 | +1.540 | [+0.569, +2.484] | 7:2 |
| livE5-stock|dev | 1 | +0.576 | [+0.576, +0.576] | 1:0 |
| livE5-bare|dev | 1 | +0.320 | [+0.320, +0.320] | 1:0 |
| livE5-prior|dev | 1 | +0.489 | [+0.489, +0.489] | 1:0 |
| livE5-ace|dev | 1 | +0.323 | [+0.323, +0.323] | 1:0 |
| livKF-bare|holdout | 16 | +1.584 | [+1.056, +2.151] | 14:2 |
| livKF-prior|holdout | 16 | +1.555 | [+1.026, +2.139] | 15:1 |
| livKF-ace|holdout | 16 | +1.677 | [+1.101, +2.314] | 14:2 |
| livKF-aceseed|holdout | 16 | +0.993 | [+0.336, +1.711] | 12:4 |
| livKF-stock|holdout | 16 | +1.783 | [+1.227, +2.406] | 15:0 |
| livKF-aceKF|holdout | 16 | +0.225 | [-0.263, +0.783] | 8:8 |
| livEng-stock|holdout | 16 | +1.471 | [+0.903, +2.094] | 14:1 |
| livEng-bare|holdout | 16 | +1.272 | [+0.742, +1.840] | 14:2 |
| livEng-prior|holdout | 16 | +1.244 | [+0.710, +1.827] | 15:1 |
| livEng-ace|holdout | 16 | +1.365 | [+0.761, +2.016] | 13:3 |
| livEng-aceseed|holdout | 16 | +0.682 | [+0.063, +1.388] | 11:5 |
| livP1-stock|holdout | 16 | +0.507 | [+0.052, +1.012] | 7:5 |
| livP1-aceseed|holdout | 16 | -0.283 | [-0.497, -0.076] | 2:9 |
| livP1-bare|holdout | 16 | +0.308 | [-0.194, +0.872] | 7:7 |
| livKF-livEng|holdout | 16 | +0.312 | [-0.116, +0.825] | 9:5 |
| livKF-livP1|holdout | 16 | +1.276 | [+0.690, +1.948] | 12:2 |
| livEng-livP1|holdout | 16 | +0.964 | [+0.400, +1.623] | 11:3 |
| aceseed-ace|holdout | 16 | +0.684 | [+0.137, +1.318] | 6:5 |
| aceKF-ace|holdout | 16 | +1.452 | [+0.836, +2.094] | 12:2 |
| aceKF-aceseed|holdout | 16 | +0.768 | [+0.228, +1.391] | 9:4 |
| ace-bare|holdout | 16 | -0.093 | [-0.262, +0.066] | 6:7 |
| prior-bare|holdout | 16 | +0.028 | [-0.067, +0.114] | 9:4 |
| stock-bare|holdout | 16 | -0.199 | [-0.335, -0.077] | 3:10 |
| stock-ace|holdout | 16 | -0.106 | [-0.221, +0.021] | 1:10 |
| aceseed-bare|holdout | 16 | +0.591 | [+0.004, +1.255] | 8:6 |
| aceKF-bare|holdout | 16 | +1.359 | [+0.770, +1.972] | 13:2 |
| livE5-stock|holdout | 5 | +0.027 | [-0.003, +0.063] | 1:0 |
| livE5-bare|holdout | 5 | -0.126 | [-0.361, +0.102] | 1:2 |
| livE5-prior|holdout | 5 | -0.024 | [-0.366, +0.264] | 2:1 |
| livE5-ace|holdout | 5 | -0.018 | [-0.378, +0.366] | 1:3 |
| livKF-bare|p1_applicable | 14 | +2.160 | [+1.619, +2.676] | 13:1 |
| livKF-prior|p1_applicable | 14 | +1.984 | [+1.377, +2.558] | 12:2 |
| livKF-ace|p1_applicable | 14 | +1.876 | [+1.197, +2.485] | 12:2 |
| livKF-aceseed|p1_applicable | 14 | +0.098 | [-0.329, +0.588] | 7:5 |
| livKF-stock|p1_applicable | 14 | +2.159 | [+1.626, +2.660] | 13:0 |
| livKF-aceKF|p1_applicable | 14 | +0.127 | [-0.337, +0.738] | 4:8 |
| livEng-stock|p1_applicable | 14 | +1.852 | [+1.193, +2.502] | 11:3 |
| livEng-bare|p1_applicable | 14 | +1.853 | [+1.202, +2.509] | 11:2 |
| livEng-prior|p1_applicable | 14 | +1.677 | [+0.924, +2.398] | 11:3 |
| livEng-ace|p1_applicable | 14 | +1.569 | [+0.714, +2.349] | 11:3 |
| livEng-aceseed|p1_applicable | 14 | -0.209 | [-0.603, +0.210] | 5:8 |
| livP1-stock|p1_applicable | 14 | +1.927 | [+1.362, +2.488] | 12:1 |
| livP1-aceseed|p1_applicable | 14 | -0.134 | [-0.408, +0.136] | 6:6 |
| livP1-bare|p1_applicable | 14 | +1.929 | [+1.350, +2.505] | 13:1 |
| livKF-livEng|p1_applicable | 14 | +0.307 | [-0.164, +0.905] | 7:5 |
| livKF-livP1|p1_applicable | 14 | +0.231 | [-0.024, +0.611] | 5:3 |
| livEng-livP1|p1_applicable | 14 | -0.075 | [-0.409, +0.268] | 5:6 |
| aceseed-ace|p1_applicable | 14 | +1.778 | [+0.911, +2.593] | 11:3 |
| aceKF-ace|p1_applicable | 14 | +1.748 | [+0.888, +2.543] | 11:3 |
| aceKF-aceseed|p1_applicable | 14 | -0.030 | [-0.306, +0.261] | 5:7 |
| ace-bare|p1_applicable | 14 | +0.284 | [+0.009, +0.668] | 10:3 |
| prior-bare|p1_applicable | 14 | +0.177 | [+0.025, +0.383] | 8:3 |
| stock-bare|p1_applicable | 14 | +0.001 | [-0.058, +0.064] | 5:4 |
| stock-ace|p1_applicable | 14 | -0.283 | [-0.661, -0.009] | 1:11 |
| aceseed-bare|p1_applicable | 14 | +2.062 | [+1.335, +2.770] | 12:2 |
| aceKF-bare|p1_applicable | 14 | +2.033 | [+1.354, +2.702] | 13:1 |
| livE5-stock|p1_applicable | 2 | +0.054 | [-0.003, +0.112] | 1:0 |
| livE5-bare|p1_applicable | 2 | +0.142 | [+0.011, +0.273] | 1:0 |
| livE5-prior|p1_applicable | 2 | +0.215 | [-0.002, +0.433] | 1:0 |
| livE5-ace|p1_applicable | 2 | +0.369 | [-0.183, +0.920] | 1:1 |
| livKF-bare|holdout_p1 | 7 | +1.821 | [+1.041, +2.585] | 6:1 |
| livKF-prior|holdout_p1 | 7 | +1.774 | [+1.002, +2.547] | 6:1 |
| livKF-ace|holdout_p1 | 7 | +1.876 | [+1.162, +2.555] | 6:1 |
| livKF-aceseed|holdout_p1 | 7 | +0.271 | [-0.511, +1.160] | 4:3 |
| livKF-stock|holdout_p1 | 7 | +1.864 | [+1.098, +2.611] | 6:0 |
| livKF-aceKF|holdout_p1 | 7 | +0.205 | [-0.642, +1.443] | 2:5 |
| livEng-stock|holdout_p1 | 7 | +1.504 | [+0.615, +2.303] | 6:1 |
| livEng-bare|holdout_p1 | 7 | +1.461 | [+0.605, +2.239] | 6:1 |
| livEng-prior|holdout_p1 | 7 | +1.414 | [+0.583, +2.205] | 6:1 |
| livEng-ace|holdout_p1 | 7 | +1.516 | [+0.660, +2.272] | 6:1 |
| livEng-aceseed|holdout_p1 | 7 | -0.089 | [-0.742, +0.620] | 3:4 |
| livP1-stock|holdout_p1 | 7 | +1.357 | [+0.559, +2.160] | 5:1 |
| livP1-aceseed|holdout_p1 | 7 | -0.236 | [-0.620, +0.146] | 2:4 |
| livP1-bare|holdout_p1 | 7 | +1.314 | [+0.519, +2.147] | 6:1 |
| livKF-livEng|holdout_p1 | 7 | +0.360 | [-0.491, +1.538] | 3:3 |
| livKF-livP1|holdout_p1 | 7 | +0.507 | [+0.056, +1.227] | 4:1 |
| livEng-livP1|holdout_p1 | 7 | +0.147 | [-0.373, +0.678] | 3:2 |
| aceseed-ace|holdout_p1 | 7 | +1.605 | [+0.514, +2.679] | 5:2 |
| aceKF-ace|holdout_p1 | 7 | +1.671 | [+0.703, +2.620] | 6:1 |
| aceKF-aceseed|holdout_p1 | 7 | +0.066 | [-0.428, +0.580] | 3:4 |
| ace-bare|holdout_p1 | 7 | -0.055 | [-0.233, +0.078] | 3:3 |
| prior-bare|holdout_p1 | 7 | +0.047 | [-0.040, +0.134] | 4:2 |
| stock-bare|holdout_p1 | 7 | -0.043 | [-0.126, +0.040] | 2:3 |
| stock-ace|holdout_p1 | 7 | +0.011 | [-0.156, +0.258] | 1:4 |
| aceseed-bare|holdout_p1 | 7 | +1.550 | [+0.400, +2.653] | 5:2 |
| aceKF-bare|holdout_p1 | 7 | +1.616 | [+0.639, +2.589] | 6:1 |
| livE5-stock|holdout_p1 | 2 | +0.054 | [-0.003, +0.112] | 1:0 |
| livE5-bare|holdout_p1 | 2 | +0.142 | [+0.011, +0.273] | 1:0 |
| livE5-prior|holdout_p1 | 2 | +0.215 | [-0.002, +0.433] | 1:0 |
| livE5-ace|holdout_p1 | 2 | +0.369 | [-0.183, +0.920] | 1:1 |
| livKF-bare|holdout_nonp1 | 9 | +1.399 | [+0.760, +2.187] | 8:1 |
| livKF-prior|holdout_nonp1 | 9 | +1.385 | [+0.748, +2.246] | 9:0 |
| livKF-ace|holdout_nonp1 | 9 | +1.522 | [+0.713, +2.513] | 8:1 |
| livKF-aceseed|holdout_nonp1 | 9 | +1.555 | [+0.737, +2.550] | 8:1 |
| livKF-stock|holdout_nonp1 | 9 | +1.720 | [+0.986, +2.651] | 9:0 |
| livKF-aceKF|holdout_nonp1 | 9 | +0.240 | [-0.213, +0.686] | 6:3 |
| livEng-stock|holdout_nonp1 | 9 | +1.446 | [+0.718, +2.368] | 8:0 |
| livEng-bare|holdout_nonp1 | 9 | +1.125 | [+0.490, +1.908] | 8:1 |
| livEng-prior|holdout_nonp1 | 9 | +1.111 | [+0.453, +1.937] | 9:0 |
| livEng-ace|holdout_nonp1 | 9 | +1.248 | [+0.441, +2.234] | 7:2 |
| livEng-aceseed|holdout_nonp1 | 9 | +1.281 | [+0.460, +2.277] | 8:1 |
| livP1-stock|holdout_nonp1 | 9 | -0.154 | [-0.371, +0.045] | 2:4 |
| livP1-aceseed|holdout_nonp1 | 9 | -0.319 | [-0.563, -0.112] | 0:5 |
| livP1-bare|holdout_nonp1 | 9 | -0.474 | [-0.755, -0.216] | 1:6 |
| livKF-livEng|holdout_nonp1 | 9 | +0.274 | [+0.034, +0.515] | 6:2 |
| livKF-livP1|holdout_nonp1 | 9 | +1.874 | [+1.067, +2.833] | 8:1 |
| livEng-livP1|holdout_nonp1 | 9 | +1.600 | [+0.833, +2.533] | 8:1 |
| aceseed-ace|holdout_nonp1 | 9 | -0.032 | [-0.186, +0.131] | 1:3 |
| aceKF-ace|holdout_nonp1 | 9 | +1.282 | [+0.545, +2.142] | 6:1 |
| aceKF-aceseed|holdout_nonp1 | 9 | +1.314 | [+0.535, +2.201] | 6:0 |
| ace-bare|holdout_nonp1 | 9 | -0.123 | [-0.395, +0.133] | 3:4 |
| prior-bare|holdout_nonp1 | 9 | +0.014 | [-0.141, +0.150] | 5:2 |
| stock-bare|holdout_nonp1 | 9 | -0.320 | [-0.523, -0.127] | 1:7 |
| stock-ace|holdout_nonp1 | 9 | -0.197 | [-0.332, -0.085] | 0:6 |
| aceseed-bare|holdout_nonp1 | 9 | -0.155 | [-0.386, +0.056] | 3:4 |
| aceKF-bare|holdout_nonp1 | 9 | +1.159 | [+0.465, +1.914] | 7:1 |
| livE5-stock|holdout_nonp1 | 3 | +0.008 | [-0.007, +0.024] | 0:0 |
| livE5-bare|holdout_nonp1 | 3 | -0.304 | [-0.504, -0.105] | 0:2 |
| livE5-prior|holdout_nonp1 | 3 | -0.183 | [-0.567, +0.200] | 1:1 |
| livE5-ace|holdout_nonp1 | 3 | -0.275 | [-0.532, -0.018] | 0:2 |
| livKF-bare|engine_card | 25 | +1.827 | [+1.337, +2.322] | 21:3 |
| livKF-prior|engine_card | 25 | +1.791 | [+1.308, +2.289] | 22:2 |
| livKF-ace|engine_card | 25 | +1.845 | [+1.349, +2.364] | 22:3 |
| livKF-aceseed|engine_card | 25 | +0.835 | [+0.317, +1.416] | 16:7 |
| livKF-stock|engine_card | 25 | +1.972 | [+1.489, +2.474] | 23:1 |
| livKF-aceKF|engine_card | 25 | +0.353 | [-0.059, +0.814] | 12:11 |
| livEng-stock|engine_card | 25 | +1.700 | [+1.239, +2.190] | 22:2 |
| livEng-bare|engine_card | 25 | +1.554 | [+1.096, +2.026] | 21:2 |
| livEng-prior|engine_card | 25 | +1.519 | [+1.056, +2.003] | 22:2 |
| livEng-ace|engine_card | 25 | +1.573 | [+1.092, +2.076] | 21:4 |
| livEng-aceseed|engine_card | 25 | +0.562 | [+0.111, +1.065] | 16:8 |
| livP1-stock|engine_card | 25 | +1.004 | [+0.553, +1.486] | 15:5 |
| livP1-aceseed|engine_card | 25 | -0.133 | [-0.342, +0.089] | 6:11 |
| livP1-bare|engine_card | 25 | +0.859 | [+0.358, +1.384] | 13:9 |
| livKF-livEng|engine_card | 25 | +0.273 | [-0.062, +0.652] | 13:8 |
| livKF-livP1|engine_card | 25 | +0.968 | [+0.487, +1.509] | 14:5 |
| livEng-livP1|engine_card | 25 | +0.695 | [+0.279, +1.166] | 16:6 |
| aceseed-ace|engine_card | 25 | +1.011 | [+0.482, +1.570] | 12:7 |
| aceKF-ace|engine_card | 25 | +1.492 | [+0.955, +2.032] | 17:5 |
| aceKF-aceseed|engine_card | 25 | +0.482 | [+0.107, +0.911] | 12:7 |
| ace-bare|engine_card | 25 | -0.019 | [-0.146, +0.101] | 12:8 |
| prior-bare|engine_card | 25 | +0.035 | [-0.035, +0.099] | 12:6 |
| stock-bare|engine_card | 25 | -0.146 | [-0.246, -0.052] | 5:13 |
| stock-ace|engine_card | 25 | -0.127 | [-0.207, -0.041] | 2:17 |
| aceseed-bare|engine_card | 25 | +0.992 | [+0.418, +1.593] | 14:8 |
| aceKF-bare|engine_card | 25 | +1.473 | [+0.932, +2.013] | 19:4 |
| livE5-stock|engine_card | 6 | +0.118 | [+0.005, +0.290] | 2:0 |
| livE5-bare|engine_card | 6 | -0.051 | [-0.295, +0.155] | 2:2 |
| livE5-prior|engine_card | 6 | +0.062 | [-0.273, +0.335] | 3:1 |
| livE5-ace|engine_card | 6 | +0.039 | [-0.295, +0.388] | 2:3 |
| livKF-bare|no_p1_engine_card | 12 | +1.390 | [+0.674, +2.192] | 9:2 |
| livKF-prior|no_p1_engine_card | 12 | +1.392 | [+0.671, +2.215] | 10:1 |
| livKF-ace|no_p1_engine_card | 12 | +1.524 | [+0.724, +2.421] | 10:2 |
| livKF-aceseed|no_p1_engine_card | 12 | +1.690 | [+0.899, +2.575] | 10:2 |
| livKF-stock|no_p1_engine_card | 12 | +1.687 | [+0.934, +2.540] | 11:1 |
| livKF-aceKF|no_p1_engine_card | 12 | +0.665 | [+0.074, +1.382] | 9:3 |
| livEng-stock|no_p1_engine_card | 12 | +1.364 | [+0.791, +2.097] | 11:0 |
| livEng-bare|no_p1_engine_card | 12 | +1.066 | [+0.559, +1.690] | 10:1 |
| livEng-prior|no_p1_engine_card | 12 | +1.069 | [+0.547, +1.733] | 11:0 |
| livEng-ace|no_p1_engine_card | 12 | +1.200 | [+0.581, +1.966] | 10:2 |
| livEng-aceseed|no_p1_engine_card | 12 | +1.367 | [+0.733, +2.128] | 11:1 |
| livP1-stock|no_p1_engine_card | 12 | -0.079 | [-0.260, +0.086] | 4:4 |
| livP1-aceseed|no_p1_engine_card | 12 | -0.076 | [-0.382, +0.282] | 1:5 |
| livP1-bare|no_p1_engine_card | 12 | -0.377 | [-0.600, -0.165] | 1:8 |
| livKF-livEng|no_p1_engine_card | 12 | +0.323 | [-0.101, +0.805] | 7:3 |
| livKF-livP1|no_p1_engine_card | 12 | +1.766 | [+0.962, +2.657] | 10:2 |
| livEng-livP1|no_p1_engine_card | 12 | +1.443 | [+0.834, +2.183] | 11:1 |
| aceseed-ace|no_p1_engine_card | 12 | -0.167 | [-0.422, +0.042] | 1:5 |
| aceKF-ace|no_p1_engine_card | 12 | +0.858 | [+0.218, +1.620] | 6:3 |
| aceKF-aceseed|no_p1_engine_card | 12 | +1.025 | [+0.393, +1.765] | 7:1 |
| ace-bare|no_p1_engine_card | 12 | -0.134 | [-0.342, +0.068] | 3:5 |
| prior-bare|no_p1_engine_card | 12 | -0.002 | [-0.122, +0.103] | 5:3 |
| stock-bare|no_p1_engine_card | 12 | -0.298 | [-0.455, -0.141] | 1:9 |
| stock-ace|no_p1_engine_card | 12 | -0.164 | [-0.271, -0.070] | 1:7 |
| aceseed-bare|no_p1_engine_card | 12 | -0.301 | [-0.631, -0.024] | 3:6 |
| aceKF-bare|no_p1_engine_card | 12 | +0.724 | [+0.089, +1.410] | 7:3 |
| livE5-stock|no_p1_engine_card | 4 | +0.150 | [-0.002, +0.428] | 1:0 |
| livE5-bare|no_p1_engine_card | 4 | -0.148 | [-0.453, +0.158] | 1:2 |
| livE5-prior|no_p1_engine_card | 4 | -0.015 | [-0.518, +0.369] | 2:1 |
| livE5-ace|no_p1_engine_card | 4 | -0.126 | [-0.477, +0.175] | 1:2 |
| livKF-bare|holdout_ace_named | 7 | +1.496 | [+0.791, +2.242] | 6:1 |
| livKF-prior|holdout_ace_named | 7 | +1.446 | [+0.726, +2.251] | 6:1 |
| livKF-ace|holdout_ace_named | 7 | +1.570 | [+0.911, +2.263] | 6:1 |
| livKF-aceseed|holdout_ace_named | 7 | +0.949 | [+0.334, +1.632] | 6:1 |
| livKF-stock|holdout_ace_named | 7 | +1.589 | [+0.903, +2.308] | 6:0 |
| livKF-aceKF|holdout_ace_named | 7 | +0.297 | [-0.616, +1.473] | 3:4 |
| livEng-stock|holdout_ace_named | 7 | +0.991 | [+0.243, +1.771] | 6:1 |
| livEng-bare|holdout_ace_named | 7 | +0.898 | [+0.169, +1.657] | 6:1 |
| livEng-prior|holdout_ace_named | 7 | +0.848 | [+0.104, +1.655] | 6:1 |
| livEng-ace|holdout_ace_named | 7 | +0.972 | [+0.181, +1.766] | 5:2 |
| livEng-aceseed|holdout_ace_named | 7 | +0.351 | [-0.196, +0.878] | 5:2 |
| livP1-stock|holdout_ace_named | 7 | +0.418 | [-0.311, +1.296] | 3:3 |
| livP1-aceseed|holdout_ace_named | 7 | -0.222 | [-0.611, +0.148] | 2:4 |
| livP1-bare|holdout_ace_named | 7 | +0.325 | [-0.464, +1.251] | 4:3 |
| livKF-livEng|holdout_ace_named | 7 | +0.598 | [-0.320, +1.678] | 4:3 |
| livKF-livP1|holdout_ace_named | 7 | +1.171 | [+0.487, +1.906] | 6:1 |
| livEng-livP1|holdout_ace_named | 7 | +0.572 | [-0.096, +1.196] | 5:1 |
| aceseed-ace|holdout_ace_named | 7 | +0.622 | [-0.078, +1.536] | 4:3 |
| aceKF-ace|holdout_ace_named | 7 | +1.273 | [+0.448, +2.069] | 6:1 |
| aceKF-aceseed|holdout_ace_named | 7 | +0.652 | [-0.045, +1.358] | 5:2 |
| ace-bare|holdout_ace_named | 7 | -0.074 | [-0.345, +0.182] | 4:3 |
| prior-bare|holdout_ace_named | 7 | +0.050 | [-0.031, +0.127] | 4:2 |
| stock-bare|holdout_ace_named | 7 | -0.093 | [-0.298, +0.083] | 3:3 |
| stock-ace|holdout_ace_named | 7 | -0.019 | [-0.196, +0.234] | 1:4 |
| aceseed-bare|holdout_ace_named | 7 | +0.547 | [-0.167, +1.511] | 4:3 |
| aceKF-bare|holdout_ace_named | 7 | +1.199 | [+0.320, +2.070] | 6:1 |
| livE5-stock|holdout_ace_named | 2 | +0.054 | [-0.003, +0.112] | 1:0 |
| livE5-bare|holdout_ace_named | 2 | +0.142 | [+0.011, +0.273] | 1:0 |
| livE5-prior|holdout_ace_named | 2 | +0.215 | [-0.002, +0.433] | 1:0 |
| livE5-ace|holdout_ace_named | 2 | +0.369 | [-0.183, +0.920] | 1:1 |
| livKF-bare|holdout_ace_unnamed | 9 | +1.652 | [+0.942, +2.463] | 8:1 |
| livKF-prior|holdout_ace_unnamed | 9 | +1.640 | [+0.930, +2.486] | 9:0 |
| livKF-ace|holdout_ace_unnamed | 9 | +1.760 | [+0.898, +2.766] | 8:1 |
| livKF-aceseed|holdout_ace_unnamed | 9 | +1.028 | [+0.024, +2.215] | 6:3 |
| livKF-stock|holdout_ace_unnamed | 9 | +1.934 | [+1.158, +2.872] | 9:0 |
| livKF-aceKF|holdout_ace_unnamed | 9 | +0.169 | [-0.226, +0.602] | 5:4 |
| livEng-stock|holdout_ace_unnamed | 9 | +1.845 | [+1.094, +2.752] | 8:0 |
| livEng-bare|holdout_ace_unnamed | 9 | +1.563 | [+0.871, +2.345] | 8:1 |
| livEng-prior|holdout_ace_unnamed | 9 | +1.552 | [+0.865, +2.366] | 9:0 |
| livEng-ace|holdout_ace_unnamed | 9 | +1.671 | [+0.836, +2.651] | 8:1 |
| livEng-aceseed|holdout_ace_unnamed | 9 | +0.939 | [-0.040, +2.087] | 6:3 |
| livP1-stock|holdout_ace_unnamed | 9 | +0.576 | [+0.021, +1.174] | 4:2 |
| livP1-aceseed|holdout_ace_unnamed | 9 | -0.330 | [-0.564, -0.118] | 0:5 |
| livP1-bare|holdout_ace_unnamed | 9 | +0.294 | [-0.333, +0.983] | 3:4 |
| livKF-livEng|holdout_ace_unnamed | 9 | +0.089 | [-0.073, +0.245] | 5:2 |
| livKF-livP1|holdout_ace_unnamed | 9 | +1.358 | [+0.503, +2.433] | 6:1 |
| livEng-livP1|holdout_ace_unnamed | 9 | +1.269 | [+0.433, +2.310] | 6:2 |
| aceseed-ace|holdout_ace_unnamed | 9 | +0.732 | [-0.059, +1.578] | 2:2 |
| aceKF-ace|holdout_ace_unnamed | 9 | +1.591 | [+0.707, +2.537] | 6:1 |
| aceKF-aceseed|holdout_ace_unnamed | 9 | +0.859 | [+0.095, +1.817] | 4:2 |
| ace-bare|holdout_ace_unnamed | 9 | -0.108 | [-0.332, +0.083] | 2:4 |
| prior-bare|holdout_ace_unnamed | 9 | +0.012 | [-0.147, +0.152] | 5:2 |
| stock-bare|holdout_ace_unnamed | 9 | -0.282 | [-0.460, -0.130] | 0:7 |
| stock-ace|holdout_ace_unnamed | 9 | -0.174 | [-0.310, -0.067] | 0:6 |
| aceseed-bare|holdout_ace_unnamed | 9 | +0.624 | [-0.206, +1.526] | 4:3 |
| aceKF-bare|holdout_ace_unnamed | 9 | +1.483 | [+0.674, +2.332] | 7:1 |
| livE5-stock|holdout_ace_unnamed | 3 | +0.008 | [-0.007, +0.024] | 0:0 |
| livE5-bare|holdout_ace_unnamed | 3 | -0.304 | [-0.504, -0.105] | 0:2 |
| livE5-prior|holdout_ace_unnamed | 3 | -0.183 | [-0.567, +0.200] | 1:1 |
| livE5-ace|holdout_ace_unnamed | 3 | -0.275 | [-0.532, -0.018] | 0:2 |
| livKF-bare|e5_problems | 6 | +0.823 | [+0.090, +1.681] | 4:2 |
| livKF-prior|e5_problems | 6 | +0.936 | [+0.306, +1.742] | 5:1 |
| livKF-ace|e5_problems | 6 | +0.913 | [+0.097, +1.753] | 4:2 |
| livKF-aceseed|e5_problems | 6 | +0.932 | [+0.149, +1.768] | 4:2 |
| livKF-stock|e5_problems | 6 | +0.992 | [+0.268, +1.812] | 5:1 |
| livKF-aceKF|e5_problems | 6 | +0.977 | [+0.019, +2.134] | 4:2 |
| livEng-stock|e5_problems | 6 | +0.438 | [-0.110, +0.934] | 4:1 |
| livEng-bare|e5_problems | 6 | +0.269 | [-0.224, +0.712] | 4:2 |
| livEng-prior|e5_problems | 6 | +0.382 | [-0.137, +0.779] | 5:1 |
| livEng-ace|e5_problems | 6 | +0.359 | [-0.322, +0.996] | 4:2 |
| livEng-aceseed|e5_problems | 6 | +0.378 | [-0.287, +0.975] | 4:2 |
| livP1-stock|e5_problems | 6 | +0.145 | [+0.029, +0.274] | 3:0 |
| livP1-aceseed|e5_problems | 6 | +0.085 | [-0.119, +0.315] | 2:2 |
| livP1-bare|e5_problems | 6 | -0.024 | [-0.236, +0.166] | 2:2 |
| livKF-livEng|e5_problems | 6 | +0.554 | [-0.419, +1.882] | 3:2 |
| livKF-livP1|e5_problems | 6 | +0.847 | [+0.125, +1.628] | 4:2 |
| livEng-livP1|e5_problems | 6 | +0.293 | [-0.355, +0.868] | 4:2 |
| aceseed-ace|e5_problems | 6 | -0.019 | [-0.065, +0.029] | 1:3 |
| aceKF-ace|e5_problems | 6 | -0.064 | [-0.554, +0.439] | 2:3 |
| aceKF-aceseed|e5_problems | 6 | -0.044 | [-0.498, +0.411] | 2:2 |
| ace-bare|e5_problems | 6 | -0.090 | [-0.348, +0.164] | 2:2 |
| prior-bare|e5_problems | 6 | -0.113 | [-0.288, +0.056] | 1:3 |
| stock-bare|e5_problems | 6 | -0.170 | [-0.339, -0.002] | 1:3 |
| stock-ace|e5_problems | 6 | -0.079 | [-0.375, +0.235] | 1:4 |
| aceseed-bare|e5_problems | 6 | -0.110 | [-0.328, +0.108] | 2:3 |
| aceKF-bare|e5_problems | 6 | -0.154 | [-0.552, +0.243] | 3:3 |
| livE5-stock|e5_problems | 6 | +0.118 | [+0.005, +0.290] | 2:0 |
| livE5-bare|e5_problems | 6 | -0.051 | [-0.295, +0.155] | 2:2 |
| livE5-prior|e5_problems | 6 | +0.062 | [-0.273, +0.335] | 3:1 |
| livE5-ace|e5_problems | 6 | +0.039 | [-0.295, +0.388] | 2:3 |


Per problem: mean capped speedup over the k repeats (cards = deployed cards whose rule fires)

| problem | split | cards | bare | prior | ace | aceseed | stock | livP1 | livEng | livKF | aceKF | livE5 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| L2_020  | dev | E-H3obs | 1.43 | 1.45 | 1.44 | 1.44 | 1.44 | 1.44 | 3.19 | 5.96 | 1.45 |  |
| L2_037  | dev | E-H4obs,P1 | 1.28 | 1.28 | 1.46 | 4.71 | 1.28 | 4.74 | 5.09 | 4.67 | 4.72 |  |
| L2_040  | dev | E-H4obs,P1 | 1.03 | 1.21 | 1.11 | 4.37 | 1.04 | 4.43 | 4.37 | 4.44 | 4.43 |  |
| L2_058  | dev | E-H3obs | 10.00 | 10.00 | 9.49 | 7.89 | 9.56 | 9.81 | 10.00 | 10.00 | 8.92 |  |
| L2_081  | dev | E-H4obs,P1 | 1.03 | 1.03 | 1.19 | 4.37 | 1.03 | 3.68 | 3.66 | 3.67 | 4.37 |  |
| L2_088  | dev | E-H4obs,P1 | 1.21 | 1.07 | 1.32 | 4.89 | 1.06 | 3.75 | 3.56 | 3.76 | 4.13 |  |
| L3_008  | dev | E-H1,E-H5 | 1.50 | 1.33 | 1.50 | 1.40 | 1.25 | 1.44 | 2.41 | 1.06 | 0.83 | 1.82 |
| L3_044  | dev | E-H4obs,E-H6,P1 | 0.89 | 1.15 | 1.33 | 3.96 | 1.19 | 4.50 | 4.61 | 4.50 | 4.47 |  |
| L3_048  | dev | P1 | 0.74 | 2.30 | 3.58 | 1.21 | 0.83 | 1.76 | 0.63 | 1.99 | 1.05 |  |
| L3_050  | dev | E-H4obs,E-H6,P1 | 2.22 | 2.52 | 2.79 | 2.93 | 2.30 | 3.36 | 2.20 | 2.88 | 2.39 |  |
| L1_066 * | holdout | E-H1 | 1.51 | 1.68 | 2.05 | 1.70 | 1.76 | 0.94 | 1.96 | 2.99 | 4.31 |  |
| L1_078  | holdout | E-H3obs | 3.26 | 2.97 | 2.29 | 2.31 | 2.28 | 2.28 | 7.78 | 8.05 | 7.11 |  |
| L2_029  | holdout | E-H4obs,P1 | 1.00 | 1.29 | 1.03 | 4.60 | 1.01 | 3.51 | 3.38 | 3.47 | 4.23 |  |
| L2_039 * | holdout | E-H4obs,E-H5,P1 | 0.99 | 1.01 | 1.19 | 1.10 | 1.01 | 1.38 | 0.00 | 4.16 | 0.00 | 1.00 |
| L2_052  | holdout | E-H1,E-H5 | 1.40 | 1.39 | 1.37 | 1.41 | 1.38 | 1.40 | 2.17 | 2.19 | 1.76 | 1.39 |
| L2_053 * | holdout | E-H4obs,P1 | 0.94 | 0.84 | 1.05 | 4.63 | 1.01 | 4.19 | 4.19 | 4.12 | 4.40 |  |
| L2_062 * | holdout | E-H4obs,P1 | 1.26 | 1.38 | 1.32 | 0.86 | 1.07 | 0.81 | 2.71 | 1.05 | 2.24 |  |
| L2_063  | holdout | E-H4obs,P1 | 1.14 | 1.19 | 1.07 | 4.59 | 1.00 | 3.55 | 3.57 | 3.54 | 4.25 |  |
| L2_067 * | holdout | E-H1 | 2.29 | 2.48 | 1.57 | 2.13 | 1.56 | 0.86 | 2.70 | 3.41 | 2.91 |  |
| L2_069  | holdout | E-H1 | 1.83 | 1.87 | 1.78 | 1.26 | 1.58 | 1.25 | 3.84 | 3.67 | 3.28 |  |
| L2_073  | holdout | E-H1,E-H5 | 2.21 | 2.46 | 1.72 | 1.72 | 1.62 | 1.58 | 2.95 | 3.38 | 1.70 | 1.61 |
| L2_092  | holdout | E-H1 | 2.69 | 2.85 | 2.71 | 2.71 | 2.54 | 2.38 | 3.41 | 2.92 | 2.70 |  |
| L3_002 * | holdout | E-H4obs,P1 | 1.38 | 1.50 | 1.32 | 2.33 | 1.15 | 2.34 | 2.62 | 2.72 | 2.39 |  |
| L3_020  | holdout | E-H1,E-H4obs,E-H5 | 1.56 | 0.96 | 2.00 | 1.93 | 1.22 | 1.59 | 1.27 | 1.41 | 1.92 | 1.25 |
| L3_033  | holdout | E-H4obs | 1.16 | 1.39 | 1.32 | 1.35 | 1.08 | 1.36 | 1.96 | 2.47 | 2.66 |  |
| L3_047 * | holdout | E-H5,P1 | 2.44 | 2.28 | 1.80 | 1.89 | 2.60 | 2.56 | 2.91 | 2.83 | 2.97 | 2.72 |

(* = problem id named in ACE's H036 playbook)