{"block": "ra-w1-0003", "task": "L1_098", "reader": "gpt-6-astra", "rep": 1, "family": "loss", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 4.9266, "final": {"score": 4.9266, "status": "ok", "correct": true, "speedup": 4.9266, "speedup_lo95": 4.923, "rid": "rk-final|ra-w1-0003|fb1f1483e0c204108c08df1d", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.9002, "code_sha": "fb1f1483e0c204108c08df1d"}, "show_all": {"role": "S", "score": 5.136, "final": {"score": 5.136, "status": "ok", "correct": true, "speedup": 5.136, "speedup_lo95": 5.1238, "rid": "rk-final|ra-w1-0003|ac535ff9f582d86ae2b2746d", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.3026, "code_sha": "ac535ff9f582d86ae2b2746d"}, "bare2": {"role": "P2_refine", "score": 5.2007, "final": {"score": 5.2007, "status": "ok", "correct": true, "speedup": 5.2007, "speedup_lo95": 5.1622, "rid": "rk-final|ra-w1-0003|6ab171c310e56a91076d8a51", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.1996, "code_sha": "6ab171c310e56a91076d8a51"}, "attempt_then_consult": {"role": "P1", "score": 4.9266, "final": {"score": 4.9266, "status": "ok", "correct": true, "speedup": 4.9266, "speedup_lo95": 4.923, "rid": "rk-final|ra-w1-0003|fb1f1483e0c204108c08df1d", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.9002, "code_sha": "fb1f1483e0c204108c08df1d"}, "audit_then_consult": {"role": "P2_audit", "score": 5.0939, "final": {"score": 5.0939, "status": "ok", "correct": true, "speedup": 5.0939, "speedup_lo95": 5.0715, "rid": "rk-final|ra-w1-0003|01449fa645d4d3af090dff91", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.2239, "code_sha": "01449fa645d4d3af090dff91"}, "prior_then_consult": {"role": "P1", "score": 4.9266, "final": {"score": 4.9266, "status": "ok", "correct": true, "speedup": 4.9266, "speedup_lo95": 4.923, "rid": "rk-final|ra-w1-0003|fb1f1483e0c204108c08df1d", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.9002, "code_sha": "fb1f1483e0c204108c08df1d"}, "consult_on_failure": {"role": "P2_refine", "score": 5.2007, "final": {"score": 5.2007, "status": "ok", "correct": true, "speedup": 5.2007, "speedup_lo95": 5.1622, "rid": "rk-final|ra-w1-0003|6ab171c310e56a91076d8a51", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.1996, "code_sha": "6ab171c310e56a91076d8a51"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0003/P1", "status": "ok", "correct": true, "speedup": 4.9002, "speedup_lo95": 4.897, "ref_ms": 8.62717, "cand_ms": 1.76059, "score_pass": 4.9002, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 957, "tokens_out": 1154, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 48.7}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0003/P2_refine", "status": "ok", "correct": true, "speedup": 5.1996, "speedup_lo95": 5.1378, "ref_ms": 8.66893, "cand_ms": 1.66722, "score_pass": 5.1996, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1574, "tokens_out": 1572, "ctx_chars": 2354, "ctx_sha": "18bc6595448ce2cf", "wall_s": 99.7}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0003/P2_consult", "status": "ok", "correct": true, "speedup": 4.8831, "speedup_lo95": 4.8449, "ref_ms": 8.63364, "cand_ms": 1.76808, "score_pass": 4.8831, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 4781, "tokens_out": 2105, "ctx_chars": 4641, "ctx_sha": "d774dc41a65551fa", "wall_s": 189.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0003/P2_audit", "status": "ok", "correct": true, "speedup": 5.2239, "speedup_lo95": 5.198, "ref_ms": 8.64263, "cand_ms": 1.65444, "score_pass": 5.2239, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 4979, "tokens_out": 2187, "ctx_chars": 4953, "ctx_sha": "ca00aee8f1379011", "wall_s": 92.9}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0003/P2_prior", "status": "ok", "correct": true, "speedup": 4.7467, "speedup_lo95": 4.6284, "ref_ms": 8.65018, "cand_ms": 1.82236, "score_pass": 4.7467, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1690, "tokens_out": 1861, "ctx_chars": 2982, "ctx_sha": "0db57ae1bbe61a66", "wall_s": 80.5}, "S": {"role": "S", "run_dir": "ra-w1-0003/S", "status": "ok", "correct": true, "speedup": 5.3026, "speedup_lo95": 5.2972, "ref_ms": 8.62216, "cand_ms": 1.62602, "score_pass": 5.3026, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 3368, "tokens_out": 1224, "ctx_chars": 1982, "ctx_sha": "54e16caf9bef9eec", "wall_s": 53.3}}, "codes": {"P1": "fb1f1483e0c204108c08df1d", "P2_refine": "6ab171c310e56a91076d8a51", "P2_consult": "e8b72a1c5582bc689b558e97", "P2_audit": "01449fa645d4d3af090dff91", "P2_prior": "0347f3a095c8f65ff7df5046", "S": "ac535ff9f582d86ae2b2746d"}, "infra_any": false, "t_start": 1791533064.3918002, "t_end": 1791533465.9422987}
{"block": "ra-w1-0008", "task": "L1_060", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.753, "final": {"score": 1.753, "status": "ok", "correct": true, "speedup": 1.753, "speedup_lo95": 1.7515, "rid": "rk-final|ra-w1-0008|50ae30611cc1b71422fe832f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7394, "code_sha": "50ae30611cc1b71422fe832f"}, "show_all": {"role": "S", "score": 1.5641, "final": {"score": 1.5641, "status": "ok", "correct": true, "speedup": 1.5641, "speedup_lo95": 1.5625, "rid": "rk-final|ra-w1-0008|afe390c781e8d8c77347c62f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5636, "code_sha": "afe390c781e8d8c77347c62f"}, "bare2": {"role": "P2_refine", "score": 2.1325, "final": {"score": 2.1325, "status": "ok", "correct": true, "speedup": 2.1325, "speedup_lo95": 2.1297, "rid": "rk-final|ra-w1-0008|0102297374e1169d9b6f0ad4", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1284, "code_sha": "0102297374e1169d9b6f0ad4"}, "attempt_then_consult": {"role": "P1", "score": 1.753, "final": {"score": 1.753, "status": "ok", "correct": true, "speedup": 1.753, "speedup_lo95": 1.7515, "rid": "rk-final|ra-w1-0008|50ae30611cc1b71422fe832f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7394, "code_sha": "50ae30611cc1b71422fe832f"}, "audit_then_consult": {"role": "P1", "score": 1.753, "final": {"score": 1.753, "status": "ok", "correct": true, "speedup": 1.753, "speedup_lo95": 1.7515, "rid": "rk-final|ra-w1-0008|50ae30611cc1b71422fe832f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7394, "code_sha": "50ae30611cc1b71422fe832f"}, "prior_then_consult": {"role": "P1", "score": 1.753, "final": {"score": 1.753, "status": "ok", "correct": true, "speedup": 1.753, "speedup_lo95": 1.7515, "rid": "rk-final|ra-w1-0008|50ae30611cc1b71422fe832f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7394, "code_sha": "50ae30611cc1b71422fe832f"}, "consult_on_failure": {"role": "P2_refine", "score": 2.1325, "final": {"score": 2.1325, "status": "ok", "correct": true, "speedup": 2.1325, "speedup_lo95": 2.1297, "rid": "rk-final|ra-w1-0008|0102297374e1169d9b6f0ad4", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1284, "code_sha": "0102297374e1169d9b6f0ad4"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0008/P1", "status": "ok", "correct": true, "speedup": 1.7394, "speedup_lo95": 1.738, "ref_ms": 15.75651, "cand_ms": 9.05874, "score_pass": 1.7394, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1298, "tokens_out": 2839, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 160.3}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0008/P2_refine", "status": "ok", "correct": true, "speedup": 2.1284, "speedup_lo95": 2.1277, "ref_ms": 15.70745, "cand_ms": 7.38003, "score_pass": 2.1284, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2951, "tokens_out": 4200, "ctx_chars": 5738, "ctx_sha": "ee8054ae97bae11f", "wall_s": 209.9}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0008/P2_consult", "status": "ok", "correct": true, "speedup": 1.7076, "speedup_lo95": 1.7067, "ref_ms": 15.70387, "cand_ms": 9.19668, "score_pass": 1.7076, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13253, "tokens_out": 3813, "ctx_chars": 16907, "ctx_sha": "623b387725420f3d", "wall_s": 351.4}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0008/P2_audit", "status": "ok", "correct": true, "speedup": 1.7025, "speedup_lo95": 1.7016, "ref_ms": 15.68128, "cand_ms": 9.21072, "score_pass": 1.7025, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13512, "tokens_out": 4289, "ctx_chars": 17219, "ctx_sha": "41e68d5e6d5220d3", "wall_s": 283.4}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0008/P2_prior", "status": "ok", "correct": true, "speedup": 1.7293, "speedup_lo95": 1.7272, "ref_ms": 15.78592, "cand_ms": 9.1283, "score_pass": 1.7293, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3067, "tokens_out": 2594, "ctx_chars": 6366, "ctx_sha": "da006cdfa28ce79e", "wall_s": 165.4}, "S": {"role": "S", "run_dir": "ra-w1-0008/S", "status": "ok", "correct": true, "speedup": 1.5636, "speedup_lo95": 1.5621, "ref_ms": 15.78233, "cand_ms": 10.0938, "score_pass": 1.5636, "error": "", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 16695, "tokens_out": 4190, "ctx_chars": 10864, "ctx_sha": "0b6222244e19e377", "wall_s": 459.9}}, "codes": {"P1": "50ae30611cc1b71422fe832f", "P2_refine": "0102297374e1169d9b6f0ad4", "P2_consult": "5bed68e693f938607f5f2caa", "P2_audit": "7a79897d3b29a795b955a104", "P2_prior": "d95caa9d3b1fd69fef810f62", "S": "afe390c781e8d8c77347c62f"}, "infra_any": false, "t_start": 1791533064.4235048, "t_end": 1791533802.6549315}
{"block": "ra-w1-0005", "task": "L1_054", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.0821, "final": {"score": 1.0821, "status": "ok", "correct": true, "speedup": 1.0821, "speedup_lo95": 1.0806, "rid": "rk-final|ra-w1-0005|d6a54d80bd4065b0c5a278a6", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3629, "code_sha": "d6a54d80bd4065b0c5a278a6"}, "show_all": {"role": "S", "score": 0.9754, "final": {"score": 0.9754, "status": "ok", "correct": true, "speedup": 0.9754, "speedup_lo95": 0.9613, "rid": "rk-final|ra-w1-0005|7faffd1f2cb77fa4bbf0b1b7", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.157, "code_sha": "7faffd1f2cb77fa4bbf0b1b7"}, "bare2": {"role": "P2_refine", "score": 1.6912, "final": {"score": 1.6912, "status": "ok", "correct": true, "speedup": 1.6912, "speedup_lo95": 1.6855, "rid": "rk-final|ra-w1-0005|8011479c7428e77936060d45", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.0212, "code_sha": "8011479c7428e77936060d45"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.6148, "final": {"score": 1.6148, "status": "ok", "correct": true, "speedup": 1.6148, "speedup_lo95": 1.601, "rid": "rk-final|ra-w1-0005|bb8e68564c65a5cc03fa971b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9659, "code_sha": "bb8e68564c65a5cc03fa971b"}, "audit_then_consult": {"role": "P2_audit", "score": 1.8571, "final": {"score": 1.8571, "status": "ok", "correct": true, "speedup": 1.8571, "speedup_lo95": 1.8455, "rid": "rk-final|ra-w1-0005|c32ac1da7a612db574f8de2b", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.0693, "code_sha": "c32ac1da7a612db574f8de2b"}, "prior_then_consult": {"role": "P2_prior", "score": 2.2754, "final": {"score": 2.2754, "status": "ok", "correct": true, "speedup": 2.2754, "speedup_lo95": 1.8576, "rid": "rk-final|ra-w1-0005|0491e50178de75241bb7c1c9", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.8602, "code_sha": "0491e50178de75241bb7c1c9"}, "consult_on_failure": {"role": "P2_refine", "score": 1.6912, "final": {"score": 1.6912, "status": "ok", "correct": true, "speedup": 1.6912, "speedup_lo95": 1.6855, "rid": "rk-final|ra-w1-0005|8011479c7428e77936060d45", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.0212, "code_sha": "8011479c7428e77936060d45"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0005/P1", "status": "ok", "correct": true, "speedup": 1.3629, "speedup_lo95": 1.3615, "ref_ms": 5.13196, "cand_ms": 3.7656, "score_pass": 1.3629, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1278, "tokens_out": 2263, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 158.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0005/P2_refine", "status": "ok", "correct": true, "speedup": 2.0212, "speedup_lo95": 2.0186, "ref_ms": 5.14902, "cand_ms": 2.54751, "score_pass": 2.0212, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2736, "tokens_out": 2476, "ctx_chars": 4806, "ctx_sha": "54335cf160bbe696", "wall_s": 173.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0005/P2_consult", "status": "ok", "correct": true, "speedup": 1.9659, "speedup_lo95": 1.9628, "ref_ms": 5.13492, "cand_ms": 2.61201, "score_pass": 1.9659, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12732, "tokens_out": 3513, "ctx_chars": 15975, "ctx_sha": "5854481e99fa5443", "wall_s": 295.4}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0005/P2_audit", "status": "ok", "correct": true, "speedup": 2.0693, "speedup_lo95": 1.88, "ref_ms": 5.78835, "cand_ms": 2.79727, "score_pass": 2.0693, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12993, "tokens_out": 3384, "ctx_chars": 16287, "ctx_sha": "627c6cd9e83ede16", "wall_s": 356.5}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0005/P2_prior", "status": "ok", "correct": true, "speedup": 1.8602, "speedup_lo95": 1.8573, "ref_ms": 5.21285, "cand_ms": 2.80226, "score_pass": 1.8602, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2852, "tokens_out": 2401, "ctx_chars": 5434, "ctx_sha": "8dc422c5dcfd1aba", "wall_s": 237.2}, "S": {"role": "S", "run_dir": "ra-w1-0005/S", "status": "ok", "correct": true, "speedup": 1.157, "speedup_lo95": 1.149, "ref_ms": 5.14875, "cand_ms": 4.45023, "score_pass": 1.157, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9654, "tokens_out": 3013, "ctx_chars": 10864, "ctx_sha": "0b6222244e19e377", "wall_s": 232.7}}, "codes": {"P1": "d6a54d80bd4065b0c5a278a6", "P2_refine": "8011479c7428e77936060d45", "P2_consult": "bb8e68564c65a5cc03fa971b", "P2_audit": "c32ac1da7a612db574f8de2b", "P2_prior": "0491e50178de75241bb7c1c9", "S": "7faffd1f2cb77fa4bbf0b1b7"}, "infra_any": false, "t_start": 1791533064.3957489, "t_end": 1791533807.4507875}
{"block": "ra-w1-0009", "task": "L1_079", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}, "show_all": {"role": "S", "score": 0.5586, "final": {"score": 0.5586, "status": "ok", "correct": true, "speedup": 0.5586, "speedup_lo95": 0.5353, "rid": "rk-final|ra-w1-0009|b6a1b132e0a057e6b97c5bf1", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.7061, "code_sha": "b6a1b132e0a057e6b97c5bf1"}, "bare2": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}, "attempt_then_consult": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}, "audit_then_consult": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}, "prior_then_consult": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}, "consult_on_failure": {"role": "P1", "score": 4.7901, "final": {"score": 4.7901, "status": "ok", "correct": true, "speedup": 4.7901, "speedup_lo95": 4.2778, "rid": "rk-final|ra-w1-0009|5bed9a11dc364b800038e876", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7784, "code_sha": "5bed9a11dc364b800038e876"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0009/P1", "status": "ok", "correct": true, "speedup": 4.7784, "speedup_lo95": 4.2548, "ref_ms": 8.50986, "cand_ms": 1.7809, "score_pass": 4.7784, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1257, "tokens_out": 2481, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 188.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0009/P2_refine", "status": "ok", "correct": true, "speedup": 4.7577, "speedup_lo95": 4.4527, "ref_ms": 10.30083, "cand_ms": 2.1651, "score_pass": 4.7577, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2471, "tokens_out": 2150, "ctx_chars": 4623, "ctx_sha": "20ffe077222a4570", "wall_s": 274.3}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0009/P2_consult", "status": "ok", "correct": true, "speedup": 4.022, "speedup_lo95": 4.0029, "ref_ms": 8.61494, "cand_ms": 2.14197, "score_pass": 4.022, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12308, "tokens_out": 3078, "ctx_chars": 16103, "ctx_sha": "046df7b1493641a4", "wall_s": 283.7}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0009/P2_audit", "status": "ok", "correct": true, "speedup": 3.7366, "speedup_lo95": 3.7242, "ref_ms": 8.70645, "cand_ms": 2.33002, "score_pass": 3.7366, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12528, "tokens_out": 3352, "ctx_chars": 16415, "ctx_sha": "dce347f5264a2cfe", "wall_s": 342.5}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0009/P2_prior", "status": "ok", "correct": true, "speedup": 1.5493, "speedup_lo95": 1.5459, "ref_ms": 8.50775, "cand_ms": 5.49135, "score_pass": 1.5493, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2587, "tokens_out": 3632, "ctx_chars": 5251, "ctx_sha": "d02168dbb98eb0b8", "wall_s": 216.0}, "S": {"role": "S", "run_dir": "ra-w1-0009/S", "status": "ok", "correct": true, "speedup": 0.7061, "speedup_lo95": 0.607, "ref_ms": 10.28577, "cand_ms": 14.56783, "score_pass": 0.7061, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9599, "tokens_out": 2665, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 188.3}}, "codes": {"P1": "5bed9a11dc364b800038e876", "P2_refine": "69c26484516019578e3a012e", "P2_consult": "e87fd5f239a1cbe760db3eff", "P2_audit": "9a833a2aa6cd75d123197308", "P2_prior": "f0a9679983f1f4712c3cf46e", "S": "b6a1b132e0a057e6b97c5bf1"}, "infra_any": false, "t_start": 1791533064.4358995, "t_end": 1791533825.3693087}
{"block": "ra-w1-0006", "task": "L2_021", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.5608, "final": {"score": 1.5608, "status": "ok", "correct": true, "speedup": 1.5608, "speedup_lo95": 1.54, "rid": "rk-final|ra-w1-0006|291fc2ef94fa810596daec4b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7101, "code_sha": "291fc2ef94fa810596daec4b"}, "show_all": {"role": "S", "score": 1.4258, "final": {"score": 1.4258, "status": "ok", "correct": true, "speedup": 1.4258, "speedup_lo95": 1.4247, "rid": "rk-final|ra-w1-0006|b5c264003cab2f8baaba859a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5488, "code_sha": "b5c264003cab2f8baaba859a"}, "bare2": {"role": "P1", "score": 1.5608, "final": {"score": 1.5608, "status": "ok", "correct": true, "speedup": 1.5608, "speedup_lo95": 1.54, "rid": "rk-final|ra-w1-0006|291fc2ef94fa810596daec4b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7101, "code_sha": "291fc2ef94fa810596daec4b"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.6612, "final": {"score": 1.6612, "status": "ok", "correct": true, "speedup": 1.6612, "speedup_lo95": 1.6508, "rid": "rk-final|ra-w1-0006|4c015de735f497ecdde63a35", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7282, "code_sha": "4c015de735f497ecdde63a35"}, "audit_then_consult": {"role": "P1", "score": 1.5608, "final": {"score": 1.5608, "status": "ok", "correct": true, "speedup": 1.5608, "speedup_lo95": 1.54, "rid": "rk-final|ra-w1-0006|291fc2ef94fa810596daec4b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7101, "code_sha": "291fc2ef94fa810596daec4b"}, "prior_then_consult": {"role": "P2_prior", "score": 1.6666, "final": {"score": 1.6666, "status": "ok", "correct": true, "speedup": 1.6666, "speedup_lo95": 1.6633, "rid": "rk-final|ra-w1-0006|72057d83de6e8610a4b8f23c", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9564, "code_sha": "72057d83de6e8610a4b8f23c"}, "consult_on_failure": {"role": "P1", "score": 1.5608, "final": {"score": 1.5608, "status": "ok", "correct": true, "speedup": 1.5608, "speedup_lo95": 1.54, "rid": "rk-final|ra-w1-0006|291fc2ef94fa810596daec4b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7101, "code_sha": "291fc2ef94fa810596daec4b"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0006/P1", "status": "ok", "correct": true, "speedup": 1.7101, "speedup_lo95": 1.6075, "ref_ms": 11.59187, "cand_ms": 6.77835, "score_pass": 1.7101, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1087, "tokens_out": 2686, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 172.9}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0006/P2_refine", "status": "ok", "correct": true, "speedup": 1.4787, "speedup_lo95": 1.4769, "ref_ms": 11.61184, "cand_ms": 7.85299, "score_pass": 1.4787, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2852, "tokens_out": 4834, "ctx_chars": 5764, "ctx_sha": "2a488d19f6b321b5", "wall_s": 455.0}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0006/P2_consult", "status": "ok", "correct": true, "speedup": 1.7282, "speedup_lo95": 1.6562, "ref_ms": 12.1813, "cand_ms": 7.04837, "score_pass": 1.7282, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 16447, "tokens_out": 6273, "ctx_chars": 20599, "ctx_sha": "9a5591a669b744e6", "wall_s": 365.4}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0006/P2_audit", "status": "ok", "correct": true, "speedup": 1.6319, "speedup_lo95": 1.6197, "ref_ms": 11.79923, "cand_ms": 7.23042, "score_pass": 1.6319, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 7047, "tokens_out": 3551, "ctx_chars": 20911, "ctx_sha": "204691cd2b26d9e7", "wall_s": 428.8}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0006/P2_prior", "status": "ok", "correct": true, "speedup": 1.9564, "speedup_lo95": 1.9551, "ref_ms": 11.58664, "cand_ms": 5.92237, "score_pass": 1.9564, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2968, "tokens_out": 3751, "ctx_chars": 6392, "ctx_sha": "6341d5f3c048dd29", "wall_s": 267.6}, "S": {"role": "S", "run_dir": "ra-w1-0006/S", "status": "ok", "correct": true, "speedup": 1.5488, "speedup_lo95": 1.546, "ref_ms": 11.61208, "cand_ms": 7.4975, "score_pass": 1.5488, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11789, "tokens_out": 3133, "ctx_chars": 14530, "ctx_sha": "531c6735226f817a", "wall_s": 289.1}}, "codes": {"P1": "291fc2ef94fa810596daec4b", "P2_refine": "685c43c7f1355a752b96082f", "P2_consult": "4c015de735f497ecdde63a35", "P2_audit": "1abac36bf3ac644bdf5fe164", "P2_prior": "72057d83de6e8610a4b8f23c", "S": "b5c264003cab2f8baaba859a"}, "infra_any": false, "t_start": 1791533064.4016821, "t_end": 1791533944.9238467}
{"block": "ra-w1-0004", "task": "L3_013", "reader": "gpt-6-astra", "rep": 1, "family": "network", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 4.3733, "final": {"score": 4.3733, "status": "ok", "correct": true, "speedup": 4.3733, "speedup_lo95": 4.3598, "rid": "rk-final|ra-w1-0004|3ebf7ad1246560697de3c68e", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.3812, "code_sha": "3ebf7ad1246560697de3c68e"}, "show_all": {"role": "S", "score": 6.6803, "final": {"score": 6.6803, "status": "ok", "correct": true, "speedup": 6.6803, "speedup_lo95": 6.6607, "rid": "rk-final|ra-w1-0004|c465727b40b97a4f5e03fe47", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.3493, "code_sha": "c465727b40b97a4f5e03fe47"}, "bare2": {"role": "P2_refine", "score": 8.0954, "final": {"score": 8.0954, "status": "ok", "correct": true, "speedup": 8.0954, "speedup_lo95": 7.905, "rid": "rk-final|ra-w1-0004|d8d7fd69a78ffb10a2c951b3", "infra": false, "tries": 1, "error": ""}, "score_pass": 8.0871, "code_sha": "d8d7fd69a78ffb10a2c951b3"}, "attempt_then_consult": {"role": "P2_consult", "score": 8.1056, "final": {"score": 8.1056, "status": "ok", "correct": true, "speedup": 8.1056, "speedup_lo95": 8.0943, "rid": "rk-final|ra-w1-0004|58de3e58f13794dfdbe1951c", "infra": false, "tries": 1, "error": ""}, "score_pass": 8.2568, "code_sha": "58de3e58f13794dfdbe1951c"}, "audit_then_consult": {"role": "P2_audit", "score": 6.7295, "final": {"score": 6.7295, "status": "ok", "correct": true, "speedup": 6.7295, "speedup_lo95": 6.7076, "rid": "rk-final|ra-w1-0004|fe7bdc4fecebc31c939d60f9", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.8627, "code_sha": "fe7bdc4fecebc31c939d60f9"}, "prior_then_consult": {"role": "P2_prior", "score": 8.0832, "final": {"score": 8.0832, "status": "ok", "correct": true, "speedup": 8.0832, "speedup_lo95": 8.023, "rid": "rk-final|ra-w1-0004|dc76a0dacbdde9ee0e7c2306", "infra": false, "tries": 1, "error": ""}, "score_pass": 8.0441, "code_sha": "dc76a0dacbdde9ee0e7c2306"}, "consult_on_failure": {"role": "P2_refine", "score": 8.0954, "final": {"score": 8.0954, "status": "ok", "correct": true, "speedup": 8.0954, "speedup_lo95": 7.905, "rid": "rk-final|ra-w1-0004|d8d7fd69a78ffb10a2c951b3", "infra": false, "tries": 1, "error": ""}, "score_pass": 8.0871, "code_sha": "d8d7fd69a78ffb10a2c951b3"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0004/P1", "status": "ok", "correct": true, "speedup": 4.3812, "speedup_lo95": 4.3728, "ref_ms": 16.6517, "cand_ms": 3.80069, "score_pass": 4.3812, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1057, "tokens_out": 3152, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 205.7}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0004/P2_refine", "status": "ok", "correct": true, "speedup": 8.0871, "speedup_lo95": 7.9916, "ref_ms": 16.95464, "cand_ms": 2.09649, "score_pass": 8.0871, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2914, "tokens_out": 3100, "ctx_chars": 6317, "ctx_sha": "c7fc68e910b865e1", "wall_s": 278.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0004/P2_consult", "status": "ok", "correct": true, "speedup": 8.2568, "speedup_lo95": 8.2065, "ref_ms": 16.95716, "cand_ms": 2.05371, "score_pass": 8.2568, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11001, "tokens_out": 4224, "ctx_chars": 13135, "ctx_sha": "dde6dcbce78d28ef", "wall_s": 352.1}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0004/P2_audit", "status": "ok", "correct": true, "speedup": 6.8627, "speedup_lo95": 6.822, "ref_ms": 16.80011, "cand_ms": 2.44803, "score_pass": 6.8627, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11356, "tokens_out": 4483, "ctx_chars": 13447, "ctx_sha": "c57f7a21f9d00d8a", "wall_s": 332.1}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0004/P2_prior", "status": "ok", "correct": true, "speedup": 8.0441, "speedup_lo95": 8.0238, "ref_ms": 16.57664, "cand_ms": 2.06073, "score_pass": 8.0441, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3030, "tokens_out": 2974, "ctx_chars": 6945, "ctx_sha": "29f0bd706662078c", "wall_s": 214.8}, "S": {"role": "S", "run_dir": "ra-w1-0004/S", "status": "ok", "correct": true, "speedup": 6.3493, "speedup_lo95": 6.3301, "ref_ms": 16.66249, "cand_ms": 2.62432, "score_pass": 6.3493, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 6994, "tokens_out": 3494, "ctx_chars": 6513, "ctx_sha": "1aa698d3a9589f25", "wall_s": 190.4}}, "codes": {"P1": "3ebf7ad1246560697de3c68e", "P2_refine": "d8d7fd69a78ffb10a2c951b3", "P2_consult": "58de3e58f13794dfdbe1951c", "P2_audit": "fe7bdc4fecebc31c939d60f9", "P2_prior": "dc76a0dacbdde9ee0e7c2306", "S": "c465727b40b97a4f5e03fe47"}, "infra_any": false, "t_start": 1791533064.394978, "t_end": 1791533959.2600553}
{"block": "ra-w1-0002", "task": "L1_081", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.2691, "final": {"score": 1.2691, "status": "ok", "correct": true, "speedup": 1.2691, "speedup_lo95": 1.235, "rid": "rk-final|ra-w1-0002|e6fcd046526e8c9c5eb131f2", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2568, "code_sha": "e6fcd046526e8c9c5eb131f2"}, "show_all": {"role": "S", "score": 3.1311, "final": {"score": 3.1311, "status": "ok", "correct": true, "speedup": 3.1311, "speedup_lo95": 3.1241, "rid": "rk-final|ra-w1-0002|4d247d9aa3e4b5f389102111", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.1012, "code_sha": "4d247d9aa3e4b5f389102111"}, "bare2": {"role": "P2_refine", "score": 2.1354, "final": {"score": 2.1354, "status": "ok", "correct": true, "speedup": 2.1354, "speedup_lo95": 2.0996, "rid": "rk-final|ra-w1-0002|2188b33343a4a3563a14c06a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9607, "code_sha": "2188b33343a4a3563a14c06a"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.4963, "final": {"score": 2.4963, "status": "ok", "correct": true, "speedup": 2.4963, "speedup_lo95": 2.4856, "rid": "rk-final|ra-w1-0002|c3a60b96c3c3732ad7dbe69e", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.4516, "code_sha": "c3a60b96c3c3732ad7dbe69e"}, "audit_then_consult": {"role": "P2_audit", "score": 1.4961, "final": {"score": 1.4961, "status": "ok", "correct": true, "speedup": 1.4961, "speedup_lo95": 1.4711, "rid": "rk-final|ra-w1-0002|7fcfde53a26f70aadf8db8dd", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4722, "code_sha": "7fcfde53a26f70aadf8db8dd"}, "prior_then_consult": {"role": "P2_prior", "score": 2.0222, "final": {"score": 2.0222, "status": "ok", "correct": true, "speedup": 2.0222, "speedup_lo95": 1.9721, "rid": "rk-final|ra-w1-0002|e8547de536215bba65dab6a4", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1696, "code_sha": "e8547de536215bba65dab6a4"}, "consult_on_failure": {"role": "P2_refine", "score": 2.1354, "final": {"score": 2.1354, "status": "ok", "correct": true, "speedup": 2.1354, "speedup_lo95": 2.0996, "rid": "rk-final|ra-w1-0002|2188b33343a4a3563a14c06a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9607, "code_sha": "2188b33343a4a3563a14c06a"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0002/P1", "status": "ok", "correct": true, "speedup": 1.2568, "speedup_lo95": 1.2541, "ref_ms": 2.46125, "cand_ms": 1.95827, "score_pass": 1.2568, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1282, "tokens_out": 2969, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 185.5}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0002/P2_refine", "status": "ok", "correct": true, "speedup": 1.9607, "speedup_lo95": 1.9571, "ref_ms": 2.4587, "cand_ms": 1.25398, "score_pass": 1.9607, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2689, "tokens_out": 2840, "ctx_chars": 4461, "ctx_sha": "132f2626b1d7cda3", "wall_s": 219.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0002/P2_consult", "status": "ok", "correct": true, "speedup": 2.4516, "speedup_lo95": 2.446, "ref_ms": 2.70134, "cand_ms": 1.10187, "score_pass": 2.4516, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13139, "tokens_out": 4806, "ctx_chars": 15941, "ctx_sha": "4cd81eff87570f02", "wall_s": 367.6}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0002/P2_audit", "status": "ok", "correct": true, "speedup": 1.4722, "speedup_lo95": 1.4639, "ref_ms": 2.48758, "cand_ms": 1.68964, "score_pass": 1.4722, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13193, "tokens_out": 4478, "ctx_chars": 16253, "ctx_sha": "272e6ce561ae461e", "wall_s": 362.4}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0002/P2_prior", "status": "ok", "correct": true, "speedup": 2.1696, "speedup_lo95": 2.1663, "ref_ms": 2.70326, "cand_ms": 1.24594, "score_pass": 2.1696, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2805, "tokens_out": 2834, "ctx_chars": 5089, "ctx_sha": "4536d13d75b6777d", "wall_s": 165.4}, "S": {"role": "S", "run_dir": "ra-w1-0002/S", "status": "ok", "correct": true, "speedup": 3.1012, "speedup_lo95": 3.0965, "ref_ms": 2.70115, "cand_ms": 0.87099, "score_pass": 3.1012, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10113, "tokens_out": 3908, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 239.7}}, "codes": {"P1": "e6fcd046526e8c9c5eb131f2", "P2_refine": "2188b33343a4a3563a14c06a", "P2_consult": "c3a60b96c3c3732ad7dbe69e", "P2_audit": "7fcfde53a26f70aadf8db8dd", "P2_prior": "e8547de536215bba65dab6a4", "S": "4d247d9aa3e4b5f389102111"}, "infra_any": false, "t_start": 1791533064.3891652, "t_end": 1791533991.802472}
{"block": "ra-w1-0000", "task": "L1_070", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.6055, "final": {"score": 1.6055, "status": "ok", "correct": true, "speedup": 1.6055, "speedup_lo95": 1.6038, "rid": "rk-final|ra-w1-0000|a226cdef54f13a6e390025dc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6094, "code_sha": "a226cdef54f13a6e390025dc"}, "show_all": {"role": "S", "score": 0.9673, "final": {"score": 0.9673, "status": "ok", "correct": true, "speedup": 0.9673, "speedup_lo95": 0.9665, "rid": "rk-final|ra-w1-0000|4b109264f6a66623bcbbbeaf", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9641, "code_sha": "4b109264f6a66623bcbbbeaf"}, "bare2": {"role": "P1", "score": 1.6055, "final": {"score": 1.6055, "status": "ok", "correct": true, "speedup": 1.6055, "speedup_lo95": 1.6038, "rid": "rk-final|ra-w1-0000|a226cdef54f13a6e390025dc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6094, "code_sha": "a226cdef54f13a6e390025dc"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.7751, "final": {"score": 1.7751, "status": "ok", "correct": true, "speedup": 1.7751, "speedup_lo95": 1.7735, "rid": "rk-final|ra-w1-0000|51f5b3dd1dcd925801930c78", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7787, "code_sha": "51f5b3dd1dcd925801930c78"}, "audit_then_consult": {"role": "P2_audit", "score": 1.9456, "final": {"score": 1.9456, "status": "ok", "correct": true, "speedup": 1.9456, "speedup_lo95": 1.9425, "rid": "rk-final|ra-w1-0000|00b66ce128cf38b0e7ecdcc7", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9509, "code_sha": "00b66ce128cf38b0e7ecdcc7"}, "prior_then_consult": {"role": "P2_prior", "score": 1.8315, "final": {"score": 1.8315, "status": "ok", "correct": true, "speedup": 1.8315, "speedup_lo95": 1.8307, "rid": "rk-final|ra-w1-0000|764bfe0114244ea1dd5bb8cf", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.8354, "code_sha": "764bfe0114244ea1dd5bb8cf"}, "consult_on_failure": {"role": "P1", "score": 1.6055, "final": {"score": 1.6055, "status": "ok", "correct": true, "speedup": 1.6055, "speedup_lo95": 1.6038, "rid": "rk-final|ra-w1-0000|a226cdef54f13a6e390025dc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6094, "code_sha": "a226cdef54f13a6e390025dc"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0000/P1", "status": "ok", "correct": true, "speedup": 1.6094, "speedup_lo95": 1.6079, "ref_ms": 66.68458, "cand_ms": 41.43515, "score_pass": 1.6094, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1348, "tokens_out": 4080, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 267.8}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0000/P2_refine", "status": "ok", "correct": true, "speedup": 1.3381, "speedup_lo95": 1.3352, "ref_ms": 66.38105, "cand_ms": 49.60787, "score_pass": 1.3381, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2985, "tokens_out": 3492, "ctx_chars": 5736, "ctx_sha": "c0805af65775b0f8", "wall_s": 258.3}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0000/P2_consult", "status": "ok", "correct": true, "speedup": 1.7787, "speedup_lo95": 1.7764, "ref_ms": 66.55587, "cand_ms": 37.41878, "score_pass": 1.7787, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13671, "tokens_out": 3925, "ctx_chars": 17216, "ctx_sha": "01be6cc9750b05f8", "wall_s": 446.0}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0000/P2_audit", "status": "ok", "correct": true, "speedup": 1.9509, "speedup_lo95": 1.9479, "ref_ms": 66.56233, "cand_ms": 34.11959, "score_pass": 1.9509, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13905, "tokens_out": 4198, "ctx_chars": 17528, "ctx_sha": "8974c089798d927a", "wall_s": 463.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0000/P2_prior", "status": "ok", "correct": true, "speedup": 1.8354, "speedup_lo95": 1.8329, "ref_ms": 66.36722, "cand_ms": 36.15858, "score_pass": 1.8354, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3101, "tokens_out": 3685, "ctx_chars": 6364, "ctx_sha": "d0bad185cddccda3", "wall_s": 326.5}, "S": {"role": "S", "run_dir": "ra-w1-0000/S", "status": "ok", "correct": true, "speedup": 0.9641, "speedup_lo95": 0.9628, "ref_ms": 66.33687, "cand_ms": 68.80754, "score_pass": 0.9641, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10000, "tokens_out": 2687, "ctx_chars": 11175, "ctx_sha": "5e031feda794c122", "wall_s": 214.8}}, "codes": {"P1": "a226cdef54f13a6e390025dc", "P2_refine": "1a92820e5869746a58348cc7", "P2_consult": "51f5b3dd1dcd925801930c78", "P2_audit": "00b66ce128cf38b0e7ecdcc7", "P2_prior": "764bfe0114244ea1dd5bb8cf", "S": "4b109264f6a66623bcbbbeaf"}, "infra_any": false, "t_start": 1791533064.3858361, "t_end": 1791534041.5171053}
{"block": "ra-w1-0007", "task": "L2_091", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.0847, "final": {"score": 1.0847, "status": "ok", "correct": true, "speedup": 1.0847, "speedup_lo95": 1.0833, "rid": "rk-final|ra-w1-0007|62288d55ca5f5eea535db714", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2375, "code_sha": "62288d55ca5f5eea535db714"}, "show_all": {"role": "S", "score": 1.106, "final": {"score": 1.106, "status": "ok", "correct": true, "speedup": 1.106, "speedup_lo95": 1.0722, "rid": "rk-final|ra-w1-0007|a8ded7d372663d8d6fcdb7fd", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2129, "code_sha": "a8ded7d372663d8d6fcdb7fd"}, "bare2": {"role": "P2_refine", "score": 2.1991, "final": {"score": 2.1991, "status": "ok", "correct": true, "speedup": 2.1991, "speedup_lo95": 2.1948, "rid": "rk-final|ra-w1-0007|913289c1bc68f8bdc5e632b0", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.5217, "code_sha": "913289c1bc68f8bdc5e632b0"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.1165, "final": {"score": 2.1165, "status": "ok", "correct": true, "speedup": 2.1165, "speedup_lo95": 2.1151, "rid": "rk-final|ra-w1-0007|2e899f33ca916ace87aae8da", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1118, "code_sha": "2e899f33ca916ace87aae8da"}, "audit_then_consult": {"role": "P2_audit", "score": 1.5687, "final": {"score": 1.5687, "status": "ok", "correct": true, "speedup": 1.5687, "speedup_lo95": 1.5602, "rid": "rk-final|ra-w1-0007|240e494def9f28fcd57d47ec", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.562, "code_sha": "240e494def9f28fcd57d47ec"}, "prior_then_consult": {"role": "P2_prior", "score": 3.4577, "final": {"score": 3.4577, "status": "ok", "correct": true, "speedup": 3.4577, "speedup_lo95": 3.3156, "rid": "rk-final|ra-w1-0007|203ad0143d35d9a048f76166", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.3512, "code_sha": "203ad0143d35d9a048f76166"}, "consult_on_failure": {"role": "P2_refine", "score": 2.1991, "final": {"score": 2.1991, "status": "ok", "correct": true, "speedup": 2.1991, "speedup_lo95": 2.1948, "rid": "rk-final|ra-w1-0007|913289c1bc68f8bdc5e632b0", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.5217, "code_sha": "913289c1bc68f8bdc5e632b0"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0007/P1", "status": "ok", "correct": true, "speedup": 1.2375, "speedup_lo95": 1.1168, "ref_ms": 34.5846, "cand_ms": 27.94704, "score_pass": 1.2375, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1122, "tokens_out": 1247, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 156.2}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0007/P2_refine", "status": "ok", "correct": true, "speedup": 2.5217, "speedup_lo95": 2.1997, "ref_ms": 34.5131, "cand_ms": 13.68635, "score_pass": 2.5217, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1961, "tokens_out": 3812, "ctx_chars": 3087, "ctx_sha": "734bbc143db09525", "wall_s": 199.0}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0007/P2_consult", "status": "ok", "correct": true, "speedup": 2.1118, "speedup_lo95": 2.1103, "ref_ms": 34.39, "cand_ms": 16.28472, "score_pass": 2.1118, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14071, "tokens_out": 5318, "ctx_chars": 17099, "ctx_sha": "ba81f89185f49a51", "wall_s": 472.6}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0007/P2_audit", "status": "ok", "correct": true, "speedup": 1.562, "speedup_lo95": 1.5588, "ref_ms": 34.41904, "cand_ms": 22.03533, "score_pass": 1.562, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13830, "tokens_out": 4165, "ctx_chars": 17411, "ctx_sha": "4050cdbe641360f4", "wall_s": 357.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0007/P2_prior", "status": "ok", "correct": true, "speedup": 3.3512, "speedup_lo95": 3.3469, "ref_ms": 34.51149, "cand_ms": 10.29827, "score_pass": 3.3512, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2077, "tokens_out": 3504, "ctx_chars": 3715, "ctx_sha": "d995ac70d142e95a", "wall_s": 218.9}, "S": {"role": "S", "run_dir": "ra-w1-0007/S", "status": "ok", "correct": true, "speedup": 1.2129, "speedup_lo95": 1.1352, "ref_ms": 34.52271, "cand_ms": 28.46323, "score_pass": 1.2129, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10816, "tokens_out": 1455, "ctx_chars": 13707, "ctx_sha": "c2fd2e75f0a9c4ac", "wall_s": 184.3}}, "codes": {"P1": "62288d55ca5f5eea535db714", "P2_refine": "913289c1bc68f8bdc5e632b0", "P2_consult": "2e899f33ca916ace87aae8da", "P2_audit": "240e494def9f28fcd57d47ec", "P2_prior": "203ad0143d35d9a048f76166", "S": "a8ded7d372663d8d6fcdb7fd"}, "infra_any": false, "t_start": 1791533064.4052875, "t_end": 1791534088.5628326}
{"block": "ra-w1-0001", "task": "L1_073", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": true, "arms": {"bare": {"role": "P1", "score": 0.8968, "final": {"score": 0.8968, "status": "ok", "correct": true, "speedup": 0.8968, "speedup_lo95": 0.8952, "rid": "rk-final|ra-w1-0001|7adfc65773adf58fa8da586e", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9023, "code_sha": "7adfc65773adf58fa8da586e"}, "show_all": {"role": "S", "score": 0.6049, "final": {"score": 0.6049, "status": "ok", "correct": true, "speedup": 0.6049, "speedup_lo95": 0.583, "rid": "rk-final|ra-w1-0001|b81aed6c0e2c0bb221f13789", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.6054, "code_sha": "b81aed6c0e2c0bb221f13789"}, "bare2": {"role": "P2_refine", "score": 3.0782, "final": {"score": 3.0782, "status": "ok", "correct": true, "speedup": 3.0782, "speedup_lo95": 3.0647, "rid": "rk-final|ra-w1-0001|c9df2041c8cefc091569832f", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.7022, "code_sha": "c9df2041c8cefc091569832f"}, "attempt_then_consult": {"role": "P2_consult", "score": 3.6561, "final": {"score": 3.6561, "status": "ok", "correct": true, "speedup": 3.6561, "speedup_lo95": 3.6456, "rid": "rk-final|ra-w1-0001|b31571637d8171da87a8e6e3", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.6288, "code_sha": "b31571637d8171da87a8e6e3"}, "audit_then_consult": {"role": "P2_audit", "score": 5.0837, "final": {"score": 5.0837, "status": "ok", "correct": true, "speedup": 5.0837, "speedup_lo95": 5.0662, "rid": "rk-final|ra-w1-0001|1216d37e5dbed5b08bf42a67", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7551, "code_sha": "1216d37e5dbed5b08bf42a67"}, "prior_then_consult": {"role": "P2_prior", "score": 1.3828, "final": {"score": 1.3828, "status": "ok", "correct": true, "speedup": 1.3828, "speedup_lo95": 1.165, "rid": "rk-final|ra-w1-0001|5f51ce085b801c6f973b58a8", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.191, "code_sha": "5f51ce085b801c6f973b58a8"}, "consult_on_failure": {"role": "P2_consult", "score": 3.6561, "final": {"score": 3.6561, "status": "ok", "correct": true, "speedup": 3.6561, "speedup_lo95": 3.6456, "rid": "rk-final|ra-w1-0001|b31571637d8171da87a8e6e3", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.6288, "code_sha": "b31571637d8171da87a8e6e3"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0001/P1", "status": "ok", "correct": true, "speedup": 0.9023, "speedup_lo95": 0.9017, "ref_ms": 19.53969, "cand_ms": 21.65626, "score_pass": 0.9023, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1293, "tokens_out": 2646, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 207.6}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0001/P2_refine", "status": "ok", "correct": true, "speedup": 3.7022, "speedup_lo95": 3.1713, "ref_ms": 19.70153, "cand_ms": 5.32152, "score_pass": 3.7022, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3023, "tokens_out": 3429, "ctx_chars": 6412, "ctx_sha": "9daa6f4eed5f947b", "wall_s": 281.2}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0001/P2_consult", "status": "ok", "correct": true, "speedup": 3.6288, "speedup_lo95": 3.6243, "ref_ms": 19.60889, "cand_ms": 5.40362, "score_pass": 3.6288, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 6102, "tokens_out": 2922, "ctx_chars": 17892, "ctx_sha": "f6b7780ec4c7841b", "wall_s": 198.0}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0001/P2_audit", "status": "ok", "correct": true, "speedup": 4.7551, "speedup_lo95": 4.0994, "ref_ms": 19.57335, "cand_ms": 4.11629, "score_pass": 4.7551, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14161, "tokens_out": 4691, "ctx_chars": 18204, "ctx_sha": "71bcf1041533a6fe", "wall_s": 748.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0001/P2_prior", "status": "ok", "correct": true, "speedup": 1.191, "speedup_lo95": 1.1002, "ref_ms": 19.81103, "cand_ms": 16.63409, "score_pass": 1.191, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3139, "tokens_out": 4706, "ctx_chars": 7040, "ctx_sha": "6bc3a273e09173f0", "wall_s": 413.8}, "S": {"role": "S", "run_dir": "ra-w1-0001/S", "status": "ok", "correct": true, "speedup": 0.6054, "speedup_lo95": 0.5995, "ref_ms": 19.64871, "cand_ms": 32.45426, "score_pass": 0.6054, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10088, "tokens_out": 3741, "ctx_chars": 11175, "ctx_sha": "5e031feda794c122", "wall_s": 283.5}}, "codes": {"P1": "7adfc65773adf58fa8da586e", "P2_refine": "c9df2041c8cefc091569832f", "P2_consult": "b31571637d8171da87a8e6e3", "P2_audit": "1216d37e5dbed5b08bf42a67", "P2_prior": "5f51ce085b801c6f973b58a8", "S": "b81aed6c0e2c0bb221f13789"}, "infra_any": false, "t_start": 1791533064.3862019, "t_end": 1791534234.8399723}
{"block": "ra-w1-0014", "task": "L1_003", "reader": "gpt-6-astra", "rep": 1, "family": "matmul", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.5984, "final": {"score": 1.5984, "status": "ok", "correct": true, "speedup": 1.5984, "speedup_lo95": 1.5952, "rid": "rk-final|ra-w1-0014|6050f0e4c10339cd529d6aeb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5951, "code_sha": "6050f0e4c10339cd529d6aeb"}, "show_all": {"role": "S", "score": 1.514, "final": {"score": 1.514, "status": "ok", "correct": true, "speedup": 1.514, "speedup_lo95": 1.5126, "rid": "rk-final|ra-w1-0014|563d6695c13e037a98d2cf68", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5097, "code_sha": "563d6695c13e037a98d2cf68"}, "bare2": {"role": "P1", "score": 1.5984, "final": {"score": 1.5984, "status": "ok", "correct": true, "speedup": 1.5984, "speedup_lo95": 1.5952, "rid": "rk-final|ra-w1-0014|6050f0e4c10339cd529d6aeb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5951, "code_sha": "6050f0e4c10339cd529d6aeb"}, "attempt_then_consult": {"role": "P1", "score": 1.5984, "final": {"score": 1.5984, "status": "ok", "correct": true, "speedup": 1.5984, "speedup_lo95": 1.5952, "rid": "rk-final|ra-w1-0014|6050f0e4c10339cd529d6aeb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5951, "code_sha": "6050f0e4c10339cd529d6aeb"}, "audit_then_consult": {"role": "P1", "score": 1.5984, "final": {"score": 1.5984, "status": "ok", "correct": true, "speedup": 1.5984, "speedup_lo95": 1.5952, "rid": "rk-final|ra-w1-0014|6050f0e4c10339cd529d6aeb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5951, "code_sha": "6050f0e4c10339cd529d6aeb"}, "prior_then_consult": {"role": "P2_prior", "score": 3.165, "final": {"score": 3.165, "status": "ok", "correct": true, "speedup": 3.165, "speedup_lo95": 3.1483, "rid": "rk-final|ra-w1-0014|2cb2a6f3db010a244df5e7dd", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.1767, "code_sha": "2cb2a6f3db010a244df5e7dd"}, "consult_on_failure": {"role": "P1", "score": 1.5984, "final": {"score": 1.5984, "status": "ok", "correct": true, "speedup": 1.5984, "speedup_lo95": 1.5952, "rid": "rk-final|ra-w1-0014|6050f0e4c10339cd529d6aeb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5951, "code_sha": "6050f0e4c10339cd529d6aeb"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0014/P1", "status": "ok", "correct": true, "speedup": 1.5951, "speedup_lo95": 1.5935, "ref_ms": 14.5339, "cand_ms": 9.11167, "score_pass": 1.5951, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1027, "tokens_out": 1138, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 92.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0014/P2_refine", "status": "ok", "correct": true, "speedup": 0.4423, "speedup_lo95": 0.4417, "ref_ms": 14.51806, "cand_ms": 32.82487, "score_pass": 0.4423, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1887, "tokens_out": 1704, "ctx_chars": 2812, "ctx_sha": "f887aef38ad07ef3", "wall_s": 156.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0014/P2_consult", "status": "ok", "correct": true, "speedup": 0.3852, "speedup_lo95": 0.3847, "ref_ms": 14.53858, "cand_ms": 37.74256, "score_pass": 0.3852, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10248, "tokens_out": 2096, "ctx_chars": 13501, "ctx_sha": "1873e3cafb9d0c8e", "wall_s": 206.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0014/P2_audit", "status": "ok", "correct": true, "speedup": 1.513, "speedup_lo95": 1.5117, "ref_ms": 14.548, "cand_ms": 9.61522, "score_pass": 1.513, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 4796, "tokens_out": 1416, "ctx_chars": 13813, "ctx_sha": "d87aba938fd02c8e", "wall_s": 138.9}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0014/P2_prior", "status": "ok", "correct": true, "speedup": 3.1767, "speedup_lo95": 3.1708, "ref_ms": 14.50688, "cand_ms": 4.56658, "score_pass": 3.1767, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2003, "tokens_out": 2623, "ctx_chars": 3440, "ctx_sha": "0d863bd94cc658d8", "wall_s": 191.3}, "S": {"role": "S", "run_dir": "ra-w1-0014/S", "status": "ok", "correct": true, "speedup": 1.5097, "speedup_lo95": 1.5091, "ref_ms": 14.5232, "cand_ms": 9.61982, "score_pass": 1.5097, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8423, "tokens_out": 1651, "ctx_chars": 10384, "ctx_sha": "4573cb1c62d0cdf8", "wall_s": 253.2}}, "codes": {"P1": "6050f0e4c10339cd529d6aeb", "P2_refine": "ea9e7604744dbc8b44454999", "P2_consult": "ec29640075ecc6e69df5743c", "P2_audit": "4507e0b3f49cddfbc4058ecc", "P2_prior": "2cb2a6f3db010a244df5e7dd", "S": "563d6695c13e037a98d2cf68"}, "infra_any": false, "t_start": 1791533944.9311566, "t_end": 1791534351.8958335}
{"block": "ra-w1-0016", "task": "L2_010", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}, "show_all": {"role": "S", "score": 1.0025, "final": {"score": 1.0025, "status": "ok", "correct": true, "speedup": 1.0025, "speedup_lo95": 0.9775, "rid": "rk-final|ra-w1-0016|26d97a4bdd548060c38f0697", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0082, "code_sha": "26d97a4bdd548060c38f0697"}, "bare2": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}, "attempt_then_consult": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}, "audit_then_consult": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}, "prior_then_consult": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}, "consult_on_failure": {"role": "P1", "score": 1.2835, "final": {"score": 1.2835, "status": "ok", "correct": true, "speedup": 1.2835, "speedup_lo95": 1.2801, "rid": "rk-final|ra-w1-0016|513cb45668d3dc61ea096941", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2873, "code_sha": "513cb45668d3dc61ea096941"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0016/P1", "status": "ok", "correct": true, "speedup": 1.2873, "speedup_lo95": 1.2866, "ref_ms": 25.43943, "cand_ms": 19.76229, "score_pass": 1.2873, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1165, "tokens_out": 2138, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 160.2}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0016/P2_refine", "status": "ok", "correct": true, "speedup": 0.2694, "speedup_lo95": 0.2693, "ref_ms": 25.39473, "cand_ms": 94.26379, "score_pass": 0.2694, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2527, "tokens_out": 4435, "ctx_chars": 4993, "ctx_sha": "d8d3748fca21b130", "wall_s": 112.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0016/P2_consult", "status": "ok", "correct": true, "speedup": 1.148, "speedup_lo95": 1.1469, "ref_ms": 25.4233, "cand_ms": 22.14535, "score_pass": 1.148, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 6508, "tokens_out": 1928, "ctx_chars": 19005, "ctx_sha": "4a96959f1fd57a64", "wall_s": 89.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0016/P2_audit", "status": "ok", "correct": true, "speedup": 1.1495, "speedup_lo95": 1.1485, "ref_ms": 25.43184, "cand_ms": 22.12383, "score_pass": 1.1495, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14954, "tokens_out": 3657, "ctx_chars": 19317, "ctx_sha": "620f3f29fa84d628", "wall_s": 163.9}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0016/P2_prior", "status": "ok", "correct": true, "speedup": 0.2477, "speedup_lo95": 0.2476, "ref_ms": 25.43336, "cand_ms": 102.66636, "score_pass": 0.2477, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2643, "tokens_out": 3735, "ctx_chars": 5621, "ctx_sha": "2b72bcb49e72c52b", "wall_s": 107.6}, "S": {"role": "S", "run_dir": "ra-w1-0016/S", "status": "ok", "correct": true, "speedup": 1.0082, "speedup_lo95": 1.0052, "ref_ms": 25.57646, "cand_ms": 25.36944, "score_pass": 1.0082, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11509, "tokens_out": 2655, "ctx_chars": 13707, "ctx_sha": "c2fd2e75f0a9c4ac", "wall_s": 143.8}}, "codes": {"P1": "513cb45668d3dc61ea096941", "P2_refine": "c80d4b172f1d0ec0e55f890c", "P2_consult": "37280b2e7b827336ac41b199", "P2_audit": "7bd099b33880a68e7cb21fac", "P2_prior": "4f5fa061bf7892594d2df28e", "S": "26d97a4bdd548060c38f0697"}, "infra_any": false, "t_start": 1791533991.8029804, "t_end": 1791534438.6126149}
{"block": "ra-w1-0010", "task": "L2_021", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.5594, "final": {"score": 1.5594, "status": "ok", "correct": true, "speedup": 1.5594, "speedup_lo95": 1.5578, "rid": "rk-final|ra-w1-0010|6154c00a7f0e0e5fc9b2b756", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5727, "code_sha": "6154c00a7f0e0e5fc9b2b756"}, "show_all": {"role": "S", "score": 1.5402, "final": {"score": 1.5402, "status": "ok", "correct": true, "speedup": 1.5402, "speedup_lo95": 1.5388, "rid": "rk-final|ra-w1-0010|2f832f19741800d308fa4d9d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6042, "code_sha": "2f832f19741800d308fa4d9d"}, "bare2": {"role": "P2_refine", "score": 1.7926, "final": {"score": 1.7926, "status": "ok", "correct": true, "speedup": 1.7926, "speedup_lo95": 1.7911, "rid": "rk-final|ra-w1-0010|775a2176e1c54a6a6cf9712d", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1372, "code_sha": "775a2176e1c54a6a6cf9712d"}, "attempt_then_consult": {"role": "P1", "score": 1.5594, "final": {"score": 1.5594, "status": "ok", "correct": true, "speedup": 1.5594, "speedup_lo95": 1.5578, "rid": "rk-final|ra-w1-0010|6154c00a7f0e0e5fc9b2b756", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5727, "code_sha": "6154c00a7f0e0e5fc9b2b756"}, "audit_then_consult": {"role": "P2_audit", "score": 1.8908, "final": {"score": 1.8908, "status": "ok", "correct": true, "speedup": 1.8908, "speedup_lo95": 1.7337, "rid": "rk-final|ra-w1-0010|6900838f49e081fe14aaf9b3", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9001, "code_sha": "6900838f49e081fe14aaf9b3"}, "prior_then_consult": {"role": "P2_prior", "score": 1.5981, "final": {"score": 1.5981, "status": "ok", "correct": true, "speedup": 1.5981, "speedup_lo95": 1.5957, "rid": "rk-final|ra-w1-0010|6967c16967eeab16cf4dacf7", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.726, "code_sha": "6967c16967eeab16cf4dacf7"}, "consult_on_failure": {"role": "P2_refine", "score": 1.7926, "final": {"score": 1.7926, "status": "ok", "correct": true, "speedup": 1.7926, "speedup_lo95": 1.7911, "rid": "rk-final|ra-w1-0010|775a2176e1c54a6a6cf9712d", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1372, "code_sha": "775a2176e1c54a6a6cf9712d"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0010/P1", "status": "ok", "correct": true, "speedup": 1.5727, "speedup_lo95": 1.5686, "ref_ms": 11.67617, "cand_ms": 7.42425, "score_pass": 1.5727, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1087, "tokens_out": 2563, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 224.4}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0010/P2_refine", "status": "ok", "correct": true, "speedup": 2.1372, "speedup_lo95": 2.1355, "ref_ms": 11.60557, "cand_ms": 5.43031, "score_pass": 2.1372, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2825, "tokens_out": 3597, "ctx_chars": 5685, "ctx_sha": "14ecb1da3acf00eb", "wall_s": 265.6}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0010/P2_consult", "status": "ok", "correct": true, "speedup": 1.2404, "speedup_lo95": 1.2158, "ref_ms": 11.85295, "cand_ms": 9.55569, "score_pass": 1.2404, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 17000, "tokens_out": 6981, "ctx_chars": 20520, "ctx_sha": "0f6b9b7140ff5536", "wall_s": 592.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0010/P2_audit", "status": "ok", "correct": true, "speedup": 1.9001, "speedup_lo95": 1.7333, "ref_ms": 11.65394, "cand_ms": 6.1334, "score_pass": 1.9001, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 16300, "tokens_out": 5492, "ctx_chars": 20832, "ctx_sha": "298aa5db3f42ad09", "wall_s": 336.3}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0010/P2_prior", "status": "ok", "correct": true, "speedup": 1.726, "speedup_lo95": 1.5897, "ref_ms": 11.59149, "cand_ms": 6.71597, "score_pass": 1.726, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2941, "tokens_out": 3439, "ctx_chars": 6313, "ctx_sha": "21812701b5356ce2", "wall_s": 204.9}, "S": {"role": "S", "run_dir": "ra-w1-0010/S", "status": "ok", "correct": true, "speedup": 1.6042, "speedup_lo95": 1.5611, "ref_ms": 12.02199, "cand_ms": 7.49412, "score_pass": 1.6042, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11911, "tokens_out": 3401, "ctx_chars": 14530, "ctx_sha": "531c6735226f817a", "wall_s": 229.9}}, "codes": {"P1": "6154c00a7f0e0e5fc9b2b756", "P2_refine": "775a2176e1c54a6a6cf9712d", "P2_consult": "eed79daa883d68bbcacff092", "P2_audit": "6900838f49e081fe14aaf9b3", "P2_prior": "6967c16967eeab16cf4dacf7", "S": "2f832f19741800d308fa4d9d"}, "infra_any": false, "t_start": 1791533465.9433272, "t_end": 1791534542.1072774}
{"block": "ra-w1-0011", "task": "L2_091", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.2444, "final": {"score": 1.2444, "status": "ok", "correct": true, "speedup": 1.2444, "speedup_lo95": 1.0758, "rid": "rk-final|ra-w1-0011|f3d0d8458a00faf5d0433582", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0784, "code_sha": "f3d0d8458a00faf5d0433582"}, "show_all": {"role": "S", "score": 1.2881, "final": {"score": 1.2881, "status": "ok", "correct": true, "speedup": 1.2881, "speedup_lo95": 1.0494, "rid": "rk-final|ra-w1-0011|30b8d930ede48f106f2174f3", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0551, "code_sha": "30b8d930ede48f106f2174f3"}, "bare2": {"role": "P2_refine", "score": 2.1489, "final": {"score": 2.1489, "status": "ok", "correct": true, "speedup": 2.1489, "speedup_lo95": 2.1445, "rid": "rk-final|ra-w1-0011|a6f0f11dba2d292bbc08e406", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1615, "code_sha": "a6f0f11dba2d292bbc08e406"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.4669, "final": {"score": 1.4669, "status": "ok", "correct": true, "speedup": 1.4669, "speedup_lo95": 1.4644, "rid": "rk-final|ra-w1-0011|09daee00154e4b96d29c3383", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4736, "code_sha": "09daee00154e4b96d29c3383"}, "audit_then_consult": {"role": "P2_audit", "score": 1.6399, "final": {"score": 1.6399, "status": "ok", "correct": true, "speedup": 1.6399, "speedup_lo95": 1.575, "rid": "rk-final|ra-w1-0011|3eb01b6a5eca80e29e41ce63", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6327, "code_sha": "3eb01b6a5eca80e29e41ce63"}, "prior_then_consult": {"role": "P2_prior", "score": 1.9503, "final": {"score": 1.9503, "status": "ok", "correct": true, "speedup": 1.9503, "speedup_lo95": 1.9484, "rid": "rk-final|ra-w1-0011|f17f65a672bfd6109596d7fb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9559, "code_sha": "f17f65a672bfd6109596d7fb"}, "consult_on_failure": {"role": "P2_refine", "score": 2.1489, "final": {"score": 2.1489, "status": "ok", "correct": true, "speedup": 2.1489, "speedup_lo95": 2.1445, "rid": "rk-final|ra-w1-0011|a6f0f11dba2d292bbc08e406", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1615, "code_sha": "a6f0f11dba2d292bbc08e406"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0011/P1", "status": "ok", "correct": true, "speedup": 1.0784, "speedup_lo95": 1.0767, "ref_ms": 34.50497, "cand_ms": 31.99786, "score_pass": 1.0784, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1122, "tokens_out": 1280, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 191.5}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0011/P2_refine", "status": "ok", "correct": true, "speedup": 2.1615, "speedup_lo95": 2.1597, "ref_ms": 34.43498, "cand_ms": 15.9308, "score_pass": 2.1615, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2014, "tokens_out": 3371, "ctx_chars": 3320, "ctx_sha": "20d282053867b231", "wall_s": 211.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0011/P2_consult", "status": "ok", "correct": true, "speedup": 1.4736, "speedup_lo95": 1.4657, "ref_ms": 34.65092, "cand_ms": 23.51398, "score_pass": 1.4736, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13794, "tokens_out": 4589, "ctx_chars": 17332, "ctx_sha": "3fe1491e84b2c010", "wall_s": 359.6}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0011/P2_audit", "status": "ok", "correct": true, "speedup": 1.6327, "speedup_lo95": 1.5727, "ref_ms": 35.79475, "cand_ms": 21.92425, "score_pass": 1.6327, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14055, "tokens_out": 4542, "ctx_chars": 17644, "ctx_sha": "3cff3609a67ae437", "wall_s": 481.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0011/P2_prior", "status": "ok", "correct": true, "speedup": 1.9559, "speedup_lo95": 1.9521, "ref_ms": 34.49095, "cand_ms": 17.63403, "score_pass": 1.9559, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2130, "tokens_out": 3556, "ctx_chars": 3948, "ctx_sha": "1c002016da33ac23", "wall_s": 276.8}, "S": {"role": "S", "run_dir": "ra-w1-0011/S", "status": "ok", "correct": true, "speedup": 1.0551, "speedup_lo95": 1.0531, "ref_ms": 34.62646, "cand_ms": 32.81803, "score_pass": 1.0551, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10809, "tokens_out": 1537, "ctx_chars": 13707, "ctx_sha": "c2fd2e75f0a9c4ac", "wall_s": 287.5}}, "codes": {"P1": "f3d0d8458a00faf5d0433582", "P2_refine": "a6f0f11dba2d292bbc08e406", "P2_consult": "09daee00154e4b96d29c3383", "P2_audit": "3eb01b6a5eca80e29e41ce63", "P2_prior": "f17f65a672bfd6109596d7fb", "S": "30b8d930ede48f106f2174f3"}, "infra_any": false, "t_start": 1791533802.655407, "t_end": 1791534758.8752687}
{"block": "ra-w1-0013", "task": "L2_034", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.037, "final": {"score": 1.037, "status": "ok", "correct": true, "speedup": 1.037, "speedup_lo95": 1.0035, "rid": "rk-final|ra-w1-0013|ed1d88eca6927134f8ed5d55", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0218, "code_sha": "ed1d88eca6927134f8ed5d55"}, "show_all": {"role": "S", "score": 1.0816, "final": {"score": 1.0816, "status": "ok", "correct": true, "speedup": 1.0816, "speedup_lo95": 1.0037, "rid": "rk-final|ra-w1-0013|2f1e08d46d79ee1af16885eb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2556, "code_sha": "2f1e08d46d79ee1af16885eb"}, "bare2": {"role": "P2_refine", "score": 5.5493, "final": {"score": 5.5493, "status": "ok", "correct": true, "speedup": 5.5493, "speedup_lo95": 5.4242, "rid": "rk-final|ra-w1-0013|cc097f9d0fa1d35a4b08b16f", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.6698, "code_sha": "cc097f9d0fa1d35a4b08b16f"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.9285, "final": {"score": 1.9285, "status": "ok", "correct": true, "speedup": 1.9285, "speedup_lo95": 1.9259, "rid": "rk-final|ra-w1-0013|220a155a7d5a70a9c3abd91e", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.935, "code_sha": "220a155a7d5a70a9c3abd91e"}, "audit_then_consult": {"role": "P2_audit", "score": 1.0782, "final": {"score": 1.0782, "status": "ok", "correct": true, "speedup": 1.0782, "speedup_lo95": 0.9468, "rid": "rk-final|ra-w1-0013|01a3b97e158978a5e55f9c5a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.161, "code_sha": "01a3b97e158978a5e55f9c5a"}, "prior_then_consult": {"role": "P2_prior", "score": 4.8189, "final": {"score": 4.8189, "status": "ok", "correct": true, "speedup": 4.8189, "speedup_lo95": 4.8166, "rid": "rk-final|ra-w1-0013|dea0fdb79c1ca62757f14089", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7964, "code_sha": "dea0fdb79c1ca62757f14089"}, "consult_on_failure": {"role": "P2_refine", "score": 5.5493, "final": {"score": 5.5493, "status": "ok", "correct": true, "speedup": 5.5493, "speedup_lo95": 5.4242, "rid": "rk-final|ra-w1-0013|cc097f9d0fa1d35a4b08b16f", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.6698, "code_sha": "cc097f9d0fa1d35a4b08b16f"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0013/P1", "status": "ok", "correct": true, "speedup": 1.0218, "speedup_lo95": 1.0063, "ref_ms": 48.15251, "cand_ms": 47.12327, "score_pass": 1.0218, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1166, "tokens_out": 1096, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 205.9}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0013/P2_refine", "status": "ok", "correct": true, "speedup": 5.6698, "speedup_lo95": 5.568, "ref_ms": 48.81882, "cand_ms": 8.61039, "score_pass": 5.6698, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1920, "tokens_out": 4110, "ctx_chars": 2820, "ctx_sha": "4c4b26d29673f5a1", "wall_s": 265.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0013/P2_consult", "status": "ok", "correct": true, "speedup": 1.935, "speedup_lo95": 1.9286, "ref_ms": 48.10346, "cand_ms": 24.86008, "score_pass": 1.935, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13171, "tokens_out": 4483, "ctx_chars": 16013, "ctx_sha": "569283151f3ac35f", "wall_s": 374.5}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0013/P2_audit", "status": "ok", "correct": true, "speedup": 1.161, "speedup_lo95": 1.0283, "ref_ms": 48.08324, "cand_ms": 41.41494, "score_pass": 1.161, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13005, "tokens_out": 3766, "ctx_chars": 16325, "ctx_sha": "a063d28a702e4bf9", "wall_s": 470.2}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0013/P2_prior", "status": "ok", "correct": true, "speedup": 4.7964, "speedup_lo95": 4.7925, "ref_ms": 48.02787, "cand_ms": 10.01326, "score_pass": 4.7964, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2036, "tokens_out": 3815, "ctx_chars": 3448, "ctx_sha": "1e65f4b9682c0697", "wall_s": 373.8}, "S": {"role": "S", "run_dir": "ra-w1-0013/S", "status": "ok", "correct": true, "speedup": 1.2556, "speedup_lo95": 1.0449, "ref_ms": 48.07514, "cand_ms": 38.28759, "score_pass": 1.2556, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10355, "tokens_out": 1456, "ctx_chars": 12888, "ctx_sha": "10c1ea8230b0eef6", "wall_s": 368.3}}, "codes": {"P1": "ed1d88eca6927134f8ed5d55", "P2_refine": "cc097f9d0fa1d35a4b08b16f", "P2_consult": "220a155a7d5a70a9c3abd91e", "P2_audit": "01a3b97e158978a5e55f9c5a", "P2_prior": "dea0fdb79c1ca62757f14089", "S": "2f1e08d46d79ee1af16885eb"}, "infra_any": false, "t_start": 1791533825.3713064, "t_end": 1791534762.3828094}
{"block": "ra-w1-0018", "task": "L1_070", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}, "show_all": {"role": "S", "score": 0.967, "final": {"score": 0.967, "status": "ok", "correct": true, "speedup": 0.967, "speedup_lo95": 0.9657, "rid": "rk-final|ra-w1-0018|184c6ae2699c29cf7c3d8b4e", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9644, "code_sha": "184c6ae2699c29cf7c3d8b4e"}, "bare2": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}, "attempt_then_consult": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}, "audit_then_consult": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}, "prior_then_consult": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}, "consult_on_failure": {"role": "P1", "score": 2.1958, "final": {"score": 2.1958, "status": "ok", "correct": true, "speedup": 2.1958, "speedup_lo95": 2.1924, "rid": "rk-final|ra-w1-0018|f32b1fc9ba1bbdae20c2d81f", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2017, "code_sha": "f32b1fc9ba1bbdae20c2d81f"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0018/P1", "status": "ok", "correct": true, "speedup": 2.2017, "speedup_lo95": 2.2006, "ref_ms": 66.52067, "cand_ms": 30.21267, "score_pass": 2.2017, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1348, "tokens_out": 3349, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 209.2}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0018/P2_refine", "status": "ok", "correct": true, "speedup": 2.0884, "speedup_lo95": 2.0824, "ref_ms": 66.42041, "cand_ms": 31.80429, "score_pass": 2.0884, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3253, "tokens_out": 3390, "ctx_chars": 5936, "ctx_sha": "158e56adf4c4c594", "wall_s": 256.3}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0018/P2_consult", "status": "ok", "correct": true, "speedup": 2.0919, "speedup_lo95": 2.0895, "ref_ms": 66.39302, "cand_ms": 31.73783, "score_pass": 2.0919, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14536, "tokens_out": 5385, "ctx_chars": 17416, "ctx_sha": "00685eb94bf95e9a", "wall_s": 316.0}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0018/P2_audit", "status": "ok", "correct": true, "speedup": 1.5543, "speedup_lo95": 1.5524, "ref_ms": 66.57132, "cand_ms": 42.82984, "score_pass": 1.5543, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14989, "tokens_out": 5042, "ctx_chars": 17728, "ctx_sha": "cd299e159ec044e2", "wall_s": 355.7}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0018/P2_prior", "status": "runtime_error", "correct": false, "speedup": 0.0, "speedup_lo95": null, "ref_ms": null, "cand_ms": null, "score_pass": 0.0, "error": "OutOfResources: out of resource: shared memory, Required: 204800, Hardware limit: 166912. Reducing block sizes or `num_stages` may help.", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3369, "tokens_out": 3755, "ctx_chars": 6564, "ctx_sha": "0f84167baf68a456", "wall_s": 110.1}, "S": {"role": "S", "run_dir": "ra-w1-0018/S", "status": "ok", "correct": true, "speedup": 0.9644, "speedup_lo95": 0.9627, "ref_ms": 66.32574, "cand_ms": 68.77357, "score_pass": 0.9644, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9980, "tokens_out": 2527, "ctx_chars": 11175, "ctx_sha": "5e031feda794c122", "wall_s": 208.3}}, "codes": {"P1": "f32b1fc9ba1bbdae20c2d81f", "P2_refine": "64017afd0048d4a9fa811c56", "P2_consult": "2cfe86d281c58fd291b0c2ee", "P2_audit": "89fb281da191a343e54db3de", "P2_prior": "8d7b4177d8b0ac98fa931f37", "S": "184c6ae2699c29cf7c3d8b4e"}, "infra_any": false, "t_start": 1791534088.5638535, "t_end": 1791534816.4780033}
{"block": "ra-w1-0015", "task": "L2_031", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.204, "final": {"score": 1.204, "status": "ok", "correct": true, "speedup": 1.204, "speedup_lo95": 1.0426, "rid": "rk-final|ra-w1-0015|da43527e631aa3387b3d4a79", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1127, "code_sha": "da43527e631aa3387b3d4a79"}, "show_all": {"role": "S", "score": 1.1368, "final": {"score": 1.1368, "status": "ok", "correct": true, "speedup": 1.1368, "speedup_lo95": 1.0538, "rid": "rk-final|ra-w1-0015|547fa3bbc2f3594437faee71", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1881, "code_sha": "547fa3bbc2f3594437faee71"}, "bare2": {"role": "P2_refine", "score": 1.5526, "final": {"score": 1.5526, "status": "ok", "correct": true, "speedup": 1.5526, "speedup_lo95": 1.4838, "rid": "rk-final|ra-w1-0015|3dcfa7f5412a6741f9b979db", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5962, "code_sha": "3dcfa7f5412a6741f9b979db"}, "attempt_then_consult": {"role": "P1", "score": 1.204, "final": {"score": 1.204, "status": "ok", "correct": true, "speedup": 1.204, "speedup_lo95": 1.0426, "rid": "rk-final|ra-w1-0015|da43527e631aa3387b3d4a79", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1127, "code_sha": "da43527e631aa3387b3d4a79"}, "audit_then_consult": {"role": "P1", "score": 1.204, "final": {"score": 1.204, "status": "ok", "correct": true, "speedup": 1.204, "speedup_lo95": 1.0426, "rid": "rk-final|ra-w1-0015|da43527e631aa3387b3d4a79", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1127, "code_sha": "da43527e631aa3387b3d4a79"}, "prior_then_consult": {"role": "P1", "score": 1.204, "final": {"score": 1.204, "status": "ok", "correct": true, "speedup": 1.204, "speedup_lo95": 1.0426, "rid": "rk-final|ra-w1-0015|da43527e631aa3387b3d4a79", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1127, "code_sha": "da43527e631aa3387b3d4a79"}, "consult_on_failure": {"role": "P2_refine", "score": 1.5526, "final": {"score": 1.5526, "status": "ok", "correct": true, "speedup": 1.5526, "speedup_lo95": 1.4838, "rid": "rk-final|ra-w1-0015|3dcfa7f5412a6741f9b979db", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5962, "code_sha": "3dcfa7f5412a6741f9b979db"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0015/P1", "status": "ok", "correct": true, "speedup": 1.1127, "speedup_lo95": 1.0394, "ref_ms": 17.53599, "cand_ms": 15.76007, "score_pass": 1.1127, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1077, "tokens_out": 1056, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 173.5}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0015/P2_refine", "status": "ok", "correct": true, "speedup": 1.5962, "speedup_lo95": 1.5882, "ref_ms": 17.60537, "cand_ms": 11.02954, "score_pass": 1.5962, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1784, "tokens_out": 4167, "ctx_chars": 2721, "ctx_sha": "27aaf4046d79d02c", "wall_s": 307.6}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0015/P2_consult", "status": "ok", "correct": true, "speedup": 1.0148, "speedup_lo95": 0.8425, "ref_ms": 17.76459, "cand_ms": 17.50593, "score_pass": 1.0148, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10084, "tokens_out": 3937, "ctx_chars": 12470, "ctx_sha": "46f67c3c0ee31a2e", "wall_s": 354.8}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0015/P2_audit", "status": "ok", "correct": true, "speedup": 0.758, "speedup_lo95": 0.7522, "ref_ms": 19.01933, "cand_ms": 25.09261, "score_pass": 0.758, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9999, "tokens_out": 3072, "ctx_chars": 12782, "ctx_sha": "b2008aa3738c4a3d", "wall_s": 552.5}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0015/P2_prior", "status": "ok", "correct": true, "speedup": 0.9312, "speedup_lo95": 0.9279, "ref_ms": 17.37665, "cand_ms": 18.65971, "score_pass": 0.9312, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1900, "tokens_out": 3078, "ctx_chars": 3349, "ctx_sha": "ded54993bc3876dc", "wall_s": 285.6}, "S": {"role": "S", "run_dir": "ra-w1-0015/S", "status": "ok", "correct": true, "speedup": 1.1881, "speedup_lo95": 1.1684, "ref_ms": 17.57797, "cand_ms": 14.79561, "score_pass": 1.1881, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 7705, "tokens_out": 1158, "ctx_chars": 9444, "ctx_sha": "a1ff20f70ebfdfbd", "wall_s": 196.3}}, "codes": {"P1": "da43527e631aa3387b3d4a79", "P2_refine": "3dcfa7f5412a6741f9b979db", "P2_consult": "81039bcbef24ad067231fa26", "P2_audit": "e5838e1654d4548bf64afeee", "P2_prior": "ed6a631ea6173dfcbafc5757", "S": "547fa3bbc2f3594437faee71"}, "infra_any": false, "t_start": 1791533959.260477, "t_end": 1791534923.785381}
{"block": "ra-w1-0012", "task": "L2_093", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": true, "arms": {"bare": {"role": "P1", "score": 0.9624, "final": {"score": 0.9624, "status": "ok", "correct": true, "speedup": 0.9624, "speedup_lo95": 0.9612, "rid": "rk-final|ra-w1-0012|1fcc9e539e8e38dbc839ad90", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9663, "code_sha": "1fcc9e539e8e38dbc839ad90"}, "show_all": {"role": "S", "score": 1.1596, "final": {"score": 1.1596, "status": "ok", "correct": true, "speedup": 1.1596, "speedup_lo95": 0.9623, "rid": "rk-final|ra-w1-0012|ac65f92e1fb0a0e6abadac99", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0538, "code_sha": "ac65f92e1fb0a0e6abadac99"}, "bare2": {"role": "P2_refine", "score": 1.0161, "final": {"score": 1.0161, "status": "ok", "correct": true, "speedup": 1.0161, "speedup_lo95": 0.9777, "rid": "rk-final|ra-w1-0012|e9f89011e5016591886afe14", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.982, "code_sha": "e9f89011e5016591886afe14"}, "attempt_then_consult": {"role": "P2_consult", "score": 3.2831, "final": {"score": 3.2831, "status": "ok", "correct": true, "speedup": 3.2831, "speedup_lo95": 3.2806, "rid": "rk-final|ra-w1-0012|716541eb2cea579583b1e5da", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.3054, "code_sha": "716541eb2cea579583b1e5da"}, "audit_then_consult": {"role": "P1", "score": 0.9624, "final": {"score": 0.9624, "status": "ok", "correct": true, "speedup": 0.9624, "speedup_lo95": 0.9612, "rid": "rk-final|ra-w1-0012|1fcc9e539e8e38dbc839ad90", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9663, "code_sha": "1fcc9e539e8e38dbc839ad90"}, "prior_then_consult": {"role": "P2_prior", "score": 4.1705, "final": {"score": 4.1705, "status": "ok", "correct": true, "speedup": 4.1705, "speedup_lo95": 4.1674, "rid": "rk-final|ra-w1-0012|bff87211c513ab2d4b38bc71", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.1632, "code_sha": "bff87211c513ab2d4b38bc71"}, "consult_on_failure": {"role": "P2_consult", "score": 3.2831, "final": {"score": 3.2831, "status": "ok", "correct": true, "speedup": 3.2831, "speedup_lo95": 3.2806, "rid": "rk-final|ra-w1-0012|716541eb2cea579583b1e5da", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.3054, "code_sha": "716541eb2cea579583b1e5da"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0012/P1", "status": "ok", "correct": true, "speedup": 0.9663, "speedup_lo95": 0.9628, "ref_ms": 31.34072, "cand_ms": 32.43443, "score_pass": 0.9663, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1086, "tokens_out": 802, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 289.3}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0012/P2_refine", "status": "ok", "correct": true, "speedup": 0.982, "speedup_lo95": 0.9784, "ref_ms": 31.37103, "cand_ms": 31.94483, "score_pass": 0.982, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1554, "tokens_out": 3029, "ctx_chars": 1821, "ctx_sha": "c0c4f3816659b152", "wall_s": 237.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0012/P2_consult", "status": "ok", "correct": true, "speedup": 3.3054, "speedup_lo95": 3.2958, "ref_ms": 31.39843, "cand_ms": 9.49924, "score_pass": 3.3054, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11328, "tokens_out": 3798, "ctx_chars": 13881, "ctx_sha": "82694bccfb67953f", "wall_s": 492.7}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0012/P2_audit", "status": "ok", "correct": true, "speedup": 0.5797, "speedup_lo95": 0.5599, "ref_ms": 31.4263, "cand_ms": 54.21164, "score_pass": 0.5797, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11355, "tokens_out": 3746, "ctx_chars": 14193, "ctx_sha": "8099e84378da7789", "wall_s": 406.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0012/P2_prior", "status": "ok", "correct": true, "speedup": 4.1632, "speedup_lo95": 4.1593, "ref_ms": 31.19417, "cand_ms": 7.49282, "score_pass": 4.1632, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1670, "tokens_out": 3079, "ctx_chars": 2449, "ctx_sha": "01bd961ac3f315a6", "wall_s": 215.2}, "S": {"role": "S", "run_dir": "ra-w1-0012/S", "status": "ok", "correct": true, "speedup": 1.0538, "speedup_lo95": 0.9629, "ref_ms": 31.18352, "cand_ms": 29.59139, "score_pass": 1.0538, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9193, "tokens_out": 865, "ctx_chars": 11755, "ctx_sha": "c9bcbb50b8ac09fd", "wall_s": 282.1}}, "codes": {"P1": "1fcc9e539e8e38dbc839ad90", "P2_refine": "e9f89011e5016591886afe14", "P2_consult": "716541eb2cea579583b1e5da", "P2_audit": "247d320cd0c27eafaac858fc", "P2_prior": "bff87211c513ab2d4b38bc71", "S": "ac65f92e1fb0a0e6abadac99"}, "infra_any": false, "t_start": 1791533807.4512448, "t_end": 1791534978.8934147}
{"block": "ra-w1-0019", "task": "L1_060", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.5259, "final": {"score": 1.5259, "status": "ok", "correct": true, "speedup": 1.5259, "speedup_lo95": 1.5239, "rid": "rk-final|ra-w1-0019|b09c93dd8031cf164c20eab7", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5269, "code_sha": "b09c93dd8031cf164c20eab7"}, "show_all": {"role": "S", "score": 1.7443, "final": {"score": 1.7443, "status": "ok", "correct": true, "speedup": 1.7443, "speedup_lo95": 1.7397, "rid": "rk-final|ra-w1-0019|8502c4ed5d5ed47bb5ed36d6", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7213, "code_sha": "8502c4ed5d5ed47bb5ed36d6"}, "bare2": {"role": "P2_refine", "score": 1.4008, "final": {"score": 1.4008, "status": "ok", "correct": true, "speedup": 1.4008, "speedup_lo95": 1.3943, "rid": "rk-final|ra-w1-0019|3463d352595eb55b055885a1", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5831, "code_sha": "3463d352595eb55b055885a1"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.4399, "final": {"score": 1.4399, "status": "ok", "correct": true, "speedup": 1.4399, "speedup_lo95": 1.4288, "rid": "rk-final|ra-w1-0019|6942b9a86ebc39217a11b4b5", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5655, "code_sha": "6942b9a86ebc39217a11b4b5"}, "audit_then_consult": {"role": "P2_audit", "score": 1.6323, "final": {"score": 1.6323, "status": "ok", "correct": true, "speedup": 1.6323, "speedup_lo95": 1.6302, "rid": "rk-final|ra-w1-0019|217ed98df6952a5af0895327", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6317, "code_sha": "217ed98df6952a5af0895327"}, "prior_then_consult": {"role": "P2_prior", "score": 1.9372, "final": {"score": 1.9372, "status": "ok", "correct": true, "speedup": 1.9372, "speedup_lo95": 1.9349, "rid": "rk-final|ra-w1-0019|b66d93d6e2089c2036e63772", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9418, "code_sha": "b66d93d6e2089c2036e63772"}, "consult_on_failure": {"role": "P2_refine", "score": 1.4008, "final": {"score": 1.4008, "status": "ok", "correct": true, "speedup": 1.4008, "speedup_lo95": 1.3943, "rid": "rk-final|ra-w1-0019|3463d352595eb55b055885a1", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5831, "code_sha": "3463d352595eb55b055885a1"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0019/P1", "status": "ok", "correct": true, "speedup": 1.5269, "speedup_lo95": 1.526, "ref_ms": 15.69928, "cand_ms": 10.28174, "score_pass": 1.5269, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1298, "tokens_out": 1885, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 146.0}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0019/P2_refine", "status": "ok", "correct": true, "speedup": 1.5831, "speedup_lo95": 1.5804, "ref_ms": 15.76535, "cand_ms": 9.9586, "score_pass": 1.5831, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2775, "tokens_out": 2736, "ctx_chars": 4748, "ctx_sha": "e595bd57f966c0e5", "wall_s": 523.6}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0019/P2_consult", "status": "ok", "correct": true, "speedup": 1.5655, "speedup_lo95": 1.5636, "ref_ms": 15.70159, "cand_ms": 10.02953, "score_pass": 1.5655, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13042, "tokens_out": 4623, "ctx_chars": 15917, "ctx_sha": "236ce7548c3d7dc0", "wall_s": 392.1}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0019/P2_audit", "status": "ok", "correct": true, "speedup": 1.6317, "speedup_lo95": 1.6296, "ref_ms": 15.76823, "cand_ms": 9.66377, "score_pass": 1.6317, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13051, "tokens_out": 3847, "ctx_chars": 16229, "ctx_sha": "642f53dd03f2b3a3", "wall_s": 360.4}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0019/P2_prior", "status": "ok", "correct": true, "speedup": 1.9418, "speedup_lo95": 1.9402, "ref_ms": 15.70197, "cand_ms": 8.08648, "score_pass": 1.9418, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2891, "tokens_out": 2946, "ctx_chars": 5376, "ctx_sha": "0853249b16527d24", "wall_s": 189.5}, "S": {"role": "S", "run_dir": "ra-w1-0019/S", "status": "ok", "correct": true, "speedup": 1.7213, "speedup_lo95": 1.7195, "ref_ms": 15.78532, "cand_ms": 9.1705, "score_pass": 1.7213, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9689, "tokens_out": 3004, "ctx_chars": 10864, "ctx_sha": "0b6222244e19e377", "wall_s": 251.4}}, "codes": {"P1": "b09c93dd8031cf164c20eab7", "P2_refine": "3463d352595eb55b055885a1", "P2_consult": "6942b9a86ebc39217a11b4b5", "P2_audit": "217ed98df6952a5af0895327", "P2_prior": "b66d93d6e2089c2036e63772", "S": "8502c4ed5d5ed47bb5ed36d6"}, "infra_any": false, "t_start": 1791534234.8405094, "t_end": 1791535125.9555023}
{"block": "ra-w1-0017", "task": "L2_060", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.2639, "final": {"score": 1.2639, "status": "ok", "correct": true, "speedup": 1.2639, "speedup_lo95": 1.2045, "rid": "rk-final|ra-w1-0017|fb11192108b45d1ee4cc999a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2677, "code_sha": "fb11192108b45d1ee4cc999a"}, "show_all": {"role": "S", "score": 1.2735, "final": {"score": 1.2735, "status": "ok", "correct": true, "speedup": 1.2735, "speedup_lo95": 1.2606, "rid": "rk-final|ra-w1-0017|a322a8aee235dbcc7b926fc4", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.172, "code_sha": "a322a8aee235dbcc7b926fc4"}, "bare2": {"role": "P2_refine", "score": 4.949, "final": {"score": 4.949, "status": "ok", "correct": true, "speedup": 4.949, "speedup_lo95": 4.9382, "rid": "rk-final|ra-w1-0017|cae301d4cee00727778773a7", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.3343, "code_sha": "cae301d4cee00727778773a7"}, "attempt_then_consult": {"role": "P2_consult", "score": 4.3387, "final": {"score": 4.3387, "status": "ok", "correct": true, "speedup": 4.3387, "speedup_lo95": 4.3267, "rid": "rk-final|ra-w1-0017|a9e7cfa06ae0987435cf02cb", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7538, "code_sha": "a9e7cfa06ae0987435cf02cb"}, "audit_then_consult": {"role": "P2_audit", "score": 4.5, "final": {"score": 4.5, "status": "ok", "correct": true, "speedup": 4.5, "speedup_lo95": 4.4282, "rid": "rk-final|ra-w1-0017|174cf251bc2307aefeaf9bd9", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.3519, "code_sha": "174cf251bc2307aefeaf9bd9"}, "prior_then_consult": {"role": "P2_prior", "score": 7.9459, "final": {"score": 7.9459, "status": "ok", "correct": true, "speedup": 7.9459, "speedup_lo95": 7.3676, "rid": "rk-final|ra-w1-0017|1a5e4f65a577a6b80ee4c96e", "infra": false, "tries": 1, "error": ""}, "score_pass": 7.293, "code_sha": "1a5e4f65a577a6b80ee4c96e"}, "consult_on_failure": {"role": "P2_refine", "score": 4.949, "final": {"score": 4.949, "status": "ok", "correct": true, "speedup": 4.949, "speedup_lo95": 4.9382, "rid": "rk-final|ra-w1-0017|cae301d4cee00727778773a7", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.3343, "code_sha": "cae301d4cee00727778773a7"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0017/P1", "status": "ok", "correct": true, "speedup": 1.2677, "speedup_lo95": 1.2631, "ref_ms": 27.27917, "cand_ms": 21.51839, "score_pass": 1.2677, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1111, "tokens_out": 2135, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 217.3}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0017/P2_refine", "status": "ok", "correct": true, "speedup": 5.3343, "speedup_lo95": 5.1024, "ref_ms": 28.74114, "cand_ms": 5.388, "score_pass": 5.3343, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2467, "tokens_out": 4523, "ctx_chars": 4964, "ctx_sha": "64e627cf6ab380e5", "wall_s": 307.9}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0017/P2_consult", "status": "ok", "correct": true, "speedup": 4.7538, "speedup_lo95": 4.7473, "ref_ms": 27.33055, "cand_ms": 5.74922, "score_pass": 4.7538, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14010, "tokens_out": 5597, "ctx_chars": 17424, "ctx_sha": "d4b0d02e0980bfc6", "wall_s": 734.9}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0017/P2_audit", "status": "ok", "correct": true, "speedup": 4.3519, "speedup_lo95": 4.2681, "ref_ms": 27.29784, "cand_ms": 6.2727, "score_pass": 4.3519, "error": "", "infra": false, "infra_reruns": 0, "calls": 4, "tokens_in": 38808, "tokens_out": 11118, "ctx_chars": 17736, "ctx_sha": "4de80a861d5ceeee", "wall_s": 645.8}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0017/P2_prior", "status": "ok", "correct": true, "speedup": 7.293, "speedup_lo95": 7.2743, "ref_ms": 27.38692, "cand_ms": 3.75521, "score_pass": 7.293, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2583, "tokens_out": 4444, "ctx_chars": 5592, "ctx_sha": "070a6fa869b6e37c", "wall_s": 322.5}, "S": {"role": "S", "run_dir": "ra-w1-0017/S", "status": "ok", "correct": true, "speedup": 1.172, "speedup_lo95": 1.1161, "ref_ms": 28.49803, "cand_ms": 24.3158, "score_pass": 1.172, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10197, "tokens_out": 2498, "ctx_chars": 12155, "ctx_sha": "1323e4e600b1300c", "wall_s": 391.2}}, "codes": {"P1": "fb11192108b45d1ee4cc999a", "P2_refine": "cae301d4cee00727778773a7", "P2_consult": "a9e7cfa06ae0987435cf02cb", "P2_audit": "174cf251bc2307aefeaf9bd9", "P2_prior": "1a5e4f65a577a6b80ee4c96e", "S": "a322a8aee235dbcc7b926fc4"}, "infra_any": false, "t_start": 1791534041.5175912, "t_end": 1791535221.752459}
{"block": "ra-w1-0022", "task": "L2_034", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.1908, "final": {"score": 1.1908, "status": "ok", "correct": true, "speedup": 1.1908, "speedup_lo95": 1.0847, "rid": "rk-final|ra-w1-0022|3ec1fde7a3bca4b1f13d1489", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0067, "code_sha": "3ec1fde7a3bca4b1f13d1489"}, "show_all": {"role": "S", "score": 1.1216, "final": {"score": 1.1216, "status": "ok", "correct": true, "speedup": 1.1216, "speedup_lo95": 1.0046, "rid": "rk-final|ra-w1-0022|79daf45948d4c1ebe8887b6d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1104, "code_sha": "79daf45948d4c1ebe8887b6d"}, "bare2": {"role": "P2_refine", "score": 5.0684, "final": {"score": 5.0684, "status": "ok", "correct": true, "speedup": 5.0684, "speedup_lo95": 5.0657, "rid": "rk-final|ra-w1-0022|9be30d3cabcba692ef907900", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.0588, "code_sha": "9be30d3cabcba692ef907900"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.9291, "final": {"score": 2.9291, "status": "ok", "correct": true, "speedup": 2.9291, "speedup_lo95": 2.9183, "rid": "rk-final|ra-w1-0022|cd3b0ee6c7273ddd1232619c", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.9218, "code_sha": "cd3b0ee6c7273ddd1232619c"}, "audit_then_consult": {"role": "P2_audit", "score": 1.9345, "final": {"score": 1.9345, "status": "ok", "correct": true, "speedup": 1.9345, "speedup_lo95": 1.9275, "rid": "rk-final|ra-w1-0022|141f0d58e9526b9c5c14221e", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9357, "code_sha": "141f0d58e9526b9c5c14221e"}, "prior_then_consult": {"role": "P2_prior", "score": 4.5643, "final": {"score": 4.5643, "status": "ok", "correct": true, "speedup": 4.5643, "speedup_lo95": 4.5588, "rid": "rk-final|ra-w1-0022|480cd0e61835021b9e7ebb64", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.5659, "code_sha": "480cd0e61835021b9e7ebb64"}, "consult_on_failure": {"role": "P2_refine", "score": 5.0684, "final": {"score": 5.0684, "status": "ok", "correct": true, "speedup": 5.0684, "speedup_lo95": 5.0657, "rid": "rk-final|ra-w1-0022|9be30d3cabcba692ef907900", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.0588, "code_sha": "9be30d3cabcba692ef907900"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0022/P1", "status": "ok", "correct": true, "speedup": 1.0067, "speedup_lo95": 1.0043, "ref_ms": 48.17436, "cand_ms": 47.85577, "score_pass": 1.0067, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1166, "tokens_out": 1070, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 217.2}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0022/P2_refine", "status": "ok", "correct": true, "speedup": 5.0588, "speedup_lo95": 5.0229, "ref_ms": 47.96066, "cand_ms": 9.48059, "score_pass": 5.0588, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1914, "tokens_out": 4052, "ctx_chars": 2741, "ctx_sha": "ac4b7dd34b6d61cb", "wall_s": 225.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0022/P2_consult", "status": "ok", "correct": true, "speedup": 2.9218, "speedup_lo95": 2.9173, "ref_ms": 47.98116, "cand_ms": 16.42178, "score_pass": 2.9218, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13209, "tokens_out": 4613, "ctx_chars": 15934, "ctx_sha": "db2f9216c3d850ea", "wall_s": 358.1}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0022/P2_audit", "status": "ok", "correct": true, "speedup": 1.9357, "speedup_lo95": 1.9333, "ref_ms": 48.19871, "cand_ms": 24.90023, "score_pass": 1.9357, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 5710, "tokens_out": 2491, "ctx_chars": 16246, "ctx_sha": "b3b191928940d3a0", "wall_s": 372.2}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0022/P2_prior", "status": "ok", "correct": true, "speedup": 4.5659, "speedup_lo95": 4.5518, "ref_ms": 48.14463, "cand_ms": 10.54446, "score_pass": 4.5659, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2030, "tokens_out": 4416, "ctx_chars": 3369, "ctx_sha": "d2afcd3a67690825", "wall_s": 329.3}, "S": {"role": "S", "run_dir": "ra-w1-0022/S", "status": "ok", "correct": true, "speedup": 1.1104, "speedup_lo95": 1.0043, "ref_ms": 50.32807, "cand_ms": 45.3232, "score_pass": 1.1104, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10383, "tokens_out": 1384, "ctx_chars": 12888, "ctx_sha": "10c1ea8230b0eef6", "wall_s": 287.0}}, "codes": {"P1": "3ec1fde7a3bca4b1f13d1489", "P2_refine": "9be30d3cabcba692ef907900", "P2_consult": "cd3b0ee6c7273ddd1232619c", "P2_audit": "141f0d58e9526b9c5c14221e", "P2_prior": "480cd0e61835021b9e7ebb64", "S": "79daf45948d4c1ebe8887b6d"}, "infra_any": false, "t_start": 1791534542.1077983, "t_end": 1791535355.3540263}
{"block": "ra-w1-0021", "task": "L2_031", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.1876, "final": {"score": 1.1876, "status": "ok", "correct": true, "speedup": 1.1876, "speedup_lo95": 1.0362, "rid": "rk-final|ra-w1-0021|3d572bb018a59de33c040f08", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2161, "code_sha": "3d572bb018a59de33c040f08"}, "show_all": {"role": "S", "score": 1.0291, "final": {"score": 1.0291, "status": "ok", "correct": true, "speedup": 1.0291, "speedup_lo95": 1.0265, "rid": "rk-final|ra-w1-0021|97563b0e6e8f49ff414e71d2", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2095, "code_sha": "97563b0e6e8f49ff414e71d2"}, "bare2": {"role": "P1", "score": 1.1876, "final": {"score": 1.1876, "status": "ok", "correct": true, "speedup": 1.1876, "speedup_lo95": 1.0362, "rid": "rk-final|ra-w1-0021|3d572bb018a59de33c040f08", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2161, "code_sha": "3d572bb018a59de33c040f08"}, "attempt_then_consult": {"role": "P2_consult", "score": 0.8486, "final": {"score": 0.8486, "status": "ok", "correct": true, "speedup": 0.8486, "speedup_lo95": 0.842, "rid": "rk-final|ra-w1-0021|762052ec17e5fa8cbc3e1375", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2183, "code_sha": "762052ec17e5fa8cbc3e1375"}, "audit_then_consult": {"role": "P1", "score": 1.1876, "final": {"score": 1.1876, "status": "ok", "correct": true, "speedup": 1.1876, "speedup_lo95": 1.0362, "rid": "rk-final|ra-w1-0021|3d572bb018a59de33c040f08", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2161, "code_sha": "3d572bb018a59de33c040f08"}, "prior_then_consult": {"role": "P2_prior", "score": 1.672, "final": {"score": 1.672, "status": "ok", "correct": true, "speedup": 1.672, "speedup_lo95": 1.493, "rid": "rk-final|ra-w1-0021|015f888b9cdaef35fb9d830d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3243, "code_sha": "015f888b9cdaef35fb9d830d"}, "consult_on_failure": {"role": "P1", "score": 1.1876, "final": {"score": 1.1876, "status": "ok", "correct": true, "speedup": 1.1876, "speedup_lo95": 1.0362, "rid": "rk-final|ra-w1-0021|3d572bb018a59de33c040f08", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2161, "code_sha": "3d572bb018a59de33c040f08"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0021/P1", "status": "ok", "correct": true, "speedup": 1.2161, "speedup_lo95": 1.0767, "ref_ms": 17.53224, "cand_ms": 14.41715, "score_pass": 1.2161, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1077, "tokens_out": 1005, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 182.5}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0021/P2_refine", "status": "ok", "correct": true, "speedup": 0.8775, "speedup_lo95": 0.8722, "ref_ms": 17.50545, "cand_ms": 19.95023, "score_pass": 0.8775, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1813, "tokens_out": 3629, "ctx_chars": 2646, "ctx_sha": "6195d256bd7a7c95", "wall_s": 386.7}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0021/P2_consult", "status": "ok", "correct": true, "speedup": 1.2183, "speedup_lo95": 1.0701, "ref_ms": 19.93561, "cand_ms": 16.36366, "score_pass": 1.2183, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9811, "tokens_out": 3251, "ctx_chars": 12395, "ctx_sha": "eb4e06ee75eed8e1", "wall_s": 528.4}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0021/P2_audit", "status": "ok", "correct": true, "speedup": 1.1522, "speedup_lo95": 1.1478, "ref_ms": 17.54006, "cand_ms": 15.22262, "score_pass": 1.1522, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10400, "tokens_out": 4495, "ctx_chars": 12707, "ctx_sha": "89c8f4376e3e7bf6", "wall_s": 542.2}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0021/P2_prior", "status": "ok", "correct": true, "speedup": 1.3243, "speedup_lo95": 1.32, "ref_ms": 17.57421, "cand_ms": 13.27023, "score_pass": 1.3243, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1929, "tokens_out": 3251, "ctx_chars": 3274, "ctx_sha": "205461d416afa7f0", "wall_s": 302.7}, "S": {"role": "S", "run_dir": "ra-w1-0021/S", "status": "ok", "correct": true, "speedup": 1.2095, "speedup_lo95": 1.1308, "ref_ms": 17.27177, "cand_ms": 14.27984, "score_pass": 1.2095, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 7737, "tokens_out": 1190, "ctx_chars": 9444, "ctx_sha": "a1ff20f70ebfdfbd", "wall_s": 348.0}}, "codes": {"P1": "3d572bb018a59de33c040f08", "P2_refine": "ac7404b7679fc001779fb60d", "P2_consult": "762052ec17e5fa8cbc3e1375", "P2_audit": "bdd3db6122447dae6a8c4759", "P2_prior": "015f888b9cdaef35fb9d830d", "S": "97563b0e6e8f49ff414e71d2"}, "infra_any": false, "t_start": 1791534438.6136227, "t_end": 1791535507.4823525}
{"block": "ra-w1-0020", "task": "L2_001", "reader": "gpt-6-astra", "rep": 1, "family": "activation", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.0331, "final": {"score": 1.0331, "status": "ok", "correct": true, "speedup": 1.0331, "speedup_lo95": 0.9936, "rid": "rk-final|ra-w1-0020|d98bccd542c16a09a541291d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1128, "code_sha": "d98bccd542c16a09a541291d"}, "show_all": {"role": "S", "score": 1.0871, "final": {"score": 1.0871, "status": "ok", "correct": true, "speedup": 1.0871, "speedup_lo95": 0.9904, "rid": "rk-final|ra-w1-0020|45c06c0891fca53f7fae362e", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0624, "code_sha": "45c06c0891fca53f7fae362e"}, "bare2": {"role": "P1", "score": 1.0331, "final": {"score": 1.0331, "status": "ok", "correct": true, "speedup": 1.0331, "speedup_lo95": 0.9936, "rid": "rk-final|ra-w1-0020|d98bccd542c16a09a541291d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1128, "code_sha": "d98bccd542c16a09a541291d"}, "attempt_then_consult": {"role": "P1", "score": 1.0331, "final": {"score": 1.0331, "status": "ok", "correct": true, "speedup": 1.0331, "speedup_lo95": 0.9936, "rid": "rk-final|ra-w1-0020|d98bccd542c16a09a541291d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1128, "code_sha": "d98bccd542c16a09a541291d"}, "audit_then_consult": {"role": "P2_audit", "score": 1.0257, "final": {"score": 1.0257, "status": "ok", "correct": true, "speedup": 1.0257, "speedup_lo95": 1.0175, "rid": "rk-final|ra-w1-0020|98beb1220a6aea2bba22e0fa", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2885, "code_sha": "98beb1220a6aea2bba22e0fa"}, "prior_then_consult": {"role": "P2_prior", "score": 1.1083, "final": {"score": 1.1083, "status": "ok", "correct": true, "speedup": 1.1083, "speedup_lo95": 0.913, "rid": "rk-final|ra-w1-0020|4b09a1aaf518b54a8efd1d79", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1621, "code_sha": "4b09a1aaf518b54a8efd1d79"}, "consult_on_failure": {"role": "P1", "score": 1.0331, "final": {"score": 1.0331, "status": "ok", "correct": true, "speedup": 1.0331, "speedup_lo95": 0.9936, "rid": "rk-final|ra-w1-0020|d98bccd542c16a09a541291d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1128, "code_sha": "d98bccd542c16a09a541291d"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0020/P1", "status": "ok", "correct": true, "speedup": 1.1128, "speedup_lo95": 1.1041, "ref_ms": 15.9458, "cand_ms": 14.33001, "score_pass": 1.1128, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1008, "tokens_out": 1003, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 203.8}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0020/P2_refine", "status": "ok", "correct": true, "speedup": 0.6178, "speedup_lo95": 0.5997, "ref_ms": 18.47824, "cand_ms": 29.90819, "score_pass": 0.6178, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1629, "tokens_out": 3768, "ctx_chars": 2248, "ctx_sha": "6134d0652b1e5f16", "wall_s": 438.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0020/P2_consult", "status": "ok", "correct": true, "speedup": 0.9992, "speedup_lo95": 0.9936, "ref_ms": 15.98251, "cand_ms": 15.99575, "score_pass": 0.9992, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 9604, "tokens_out": 4407, "ctx_chars": 4535, "ctx_sha": "216b85b739ca5372", "wall_s": 544.7}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0020/P2_audit", "status": "ok", "correct": true, "speedup": 1.2885, "speedup_lo95": 1.2837, "ref_ms": 15.86695, "cand_ms": 12.31467, "score_pass": 1.2885, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 5876, "tokens_out": 3807, "ctx_chars": 4847, "ctx_sha": "91cfc5e0917a7ede", "wall_s": 329.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0020/P2_prior", "status": "ok", "correct": true, "speedup": 1.1621, "speedup_lo95": 0.9229, "ref_ms": 15.9391, "cand_ms": 13.7156, "score_pass": 1.1621, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1745, "tokens_out": 3121, "ctx_chars": 2876, "ctx_sha": "1ea3be5aea85a74d", "wall_s": 287.8}, "S": {"role": "S", "run_dir": "ra-w1-0020/S", "status": "ok", "correct": true, "speedup": 1.0624, "speedup_lo95": 0.9508, "ref_ms": 15.75679, "cand_ms": 14.8309, "score_pass": 1.0624, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 3441, "tokens_out": 1121, "ctx_chars": 1982, "ctx_sha": "54e16caf9bef9eec", "wall_s": 221.1}}, "codes": {"P1": "d98bccd542c16a09a541291d", "P2_refine": "075262c0819036fa17d66fd4", "P2_consult": "5f7e6cc9297cceb1f5d85878", "P2_audit": "98beb1220a6aea2bba22e0fa", "P2_prior": "4b09a1aaf518b54a8efd1d79", "S": "45c06c0891fca53f7fae362e"}, "infra_any": false, "t_start": 1791534351.8965807, "t_end": 1791535507.904297}
{"block": "ra-w1-0024", "task": "L2_060", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.253, "final": {"score": 1.253, "status": "ok", "correct": true, "speedup": 1.253, "speedup_lo95": 1.1929, "rid": "rk-final|ra-w1-0024|4111197904bee5d767ede263", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2639, "code_sha": "4111197904bee5d767ede263"}, "show_all": {"role": "S", "score": 1.1739, "final": {"score": 1.1739, "status": "ok", "correct": true, "speedup": 1.1739, "speedup_lo95": 1.1467, "rid": "rk-final|ra-w1-0024|0a6698ceefbe54ce9c985df6", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1722, "code_sha": "0a6698ceefbe54ce9c985df6"}, "bare2": {"role": "P1", "score": 1.253, "final": {"score": 1.253, "status": "ok", "correct": true, "speedup": 1.253, "speedup_lo95": 1.1929, "rid": "rk-final|ra-w1-0024|4111197904bee5d767ede263", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2639, "code_sha": "4111197904bee5d767ede263"}, "attempt_then_consult": {"role": "P2_consult", "score": 4.9672, "final": {"score": 4.9672, "status": "ok", "correct": true, "speedup": 4.9672, "speedup_lo95": 4.6785, "rid": "rk-final|ra-w1-0024|19b37ffaaa63bdae3345b033", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7575, "code_sha": "19b37ffaaa63bdae3345b033"}, "audit_then_consult": {"role": "P2_audit", "score": 4.6716, "final": {"score": 4.6716, "status": "ok", "correct": true, "speedup": 4.6716, "speedup_lo95": 4.2769, "rid": "rk-final|ra-w1-0024|4302bb42cb0184901add56ed", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.6003, "code_sha": "4302bb42cb0184901add56ed"}, "prior_then_consult": {"role": "P1", "score": 1.253, "final": {"score": 1.253, "status": "ok", "correct": true, "speedup": 1.253, "speedup_lo95": 1.1929, "rid": "rk-final|ra-w1-0024|4111197904bee5d767ede263", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2639, "code_sha": "4111197904bee5d767ede263"}, "consult_on_failure": {"role": "P1", "score": 1.253, "final": {"score": 1.253, "status": "ok", "correct": true, "speedup": 1.253, "speedup_lo95": 1.1929, "rid": "rk-final|ra-w1-0024|4111197904bee5d767ede263", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2639, "code_sha": "4111197904bee5d767ede263"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0024/P1", "status": "ok", "correct": true, "speedup": 1.2639, "speedup_lo95": 1.2631, "ref_ms": 27.25652, "cand_ms": 21.56618, "score_pass": 1.2639, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1111, "tokens_out": 2363, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 225.7}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0024/P2_refine", "status": "runtime_error", "correct": false, "speedup": 0.0, "speedup_lo95": null, "ref_ms": null, "cand_ms": null, "score_pass": 0.0, "error": "CompilationError: at 32:20:\n    iy = (r // WX) % HY\n    ix = r % WX\n    oz = 2 * iz + PZ\n    oy = 2 * iy + PY\n    ox = 2 * ix + PX\n\n    acc = tl.full((BLOCK,), 0.0, tl.float32)\n    for ci in tl.static_range(3):\n        for dz in tl.static_range(PZ + 1):\n            for dy in tl.static_range(PY + 1):\n                for dx in tl.static_range(PX + 1):\n                    kz: tl.constexpr = 1 - PZ + 2 * dz\n                    ^\nValueError('kz is already defined. constexpr cannot be reassigned.')", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2405, "tokens_out": 3627, "ctx_chars": 4471, "ctx_sha": "ea538eccfeaf7399", "wall_s": 448.7}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0024/P2_consult", "status": "ok", "correct": true, "speedup": 4.7575, "speedup_lo95": 4.7469, "ref_ms": 27.44367, "cand_ms": 5.76845, "score_pass": 4.7575, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13917, "tokens_out": 5312, "ctx_chars": 16931, "ctx_sha": "9e30b701b068f789", "wall_s": 435.1}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0024/P2_audit", "status": "ok", "correct": true, "speedup": 4.6003, "speedup_lo95": 4.3123, "ref_ms": 27.49062, "cand_ms": 5.97584, "score_pass": 4.6003, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14451, "tokens_out": 6398, "ctx_chars": 17243, "ctx_sha": "363ab36291ea2f04", "wall_s": 515.8}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0024/P2_prior", "status": "runtime_error", "correct": false, "speedup": 0.0, "speedup_lo95": null, "ref_ms": null, "cand_ms": null, "score_pass": 0.0, "error": "CompilationError: at 43:20:\n        for kd in tl.static_range(PD + 1):\n            for kh in tl.static_range(PH + 1):\n                for kw in tl.static_range(PW + 1):\n                    zd = d + PD - kd\n                    yh = h + PH - kh\n                    xw = w + PW - kw\n                    xv = tl.load(\n                        X + ((batch * 3 + ic) * ID + zd) * IH * IW\n                        + yh * IW + xw,\n                        valid, other=0.0,\n                    )\n                    kernel_d: tl.constexpr = 2 * kd if PD else 1\n                    ^\nValueError('kernel_d is already defined. constexpr cannot be reassigned.')", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2521, "tokens_out": 4319, "ctx_chars": 5099, "ctx_sha": "7397175f6ef57fb1", "wall_s": 184.3}, "S": {"role": "S", "run_dir": "ra-w1-0024/S", "status": "ok", "correct": true, "speedup": 1.1722, "speedup_lo95": 1.063, "ref_ms": 27.38588, "cand_ms": 23.36317, "score_pass": 1.1722, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10184, "tokens_out": 2462, "ctx_chars": 12155, "ctx_sha": "1323e4e600b1300c", "wall_s": 299.1}}, "codes": {"P1": "4111197904bee5d767ede263", "P2_refine": "116554b71f507bae30114fdf", "P2_consult": "19b37ffaaa63bdae3345b033", "P2_audit": "4302bb42cb0184901add56ed", "P2_prior": "fb0e3e94082b11c306b30546", "S": "0a6698ceefbe54ce9c985df6"}, "infra_any": false, "t_start": 1791534762.383344, "t_end": 1791535753.511639}
{"block": "ra-w1-0027", "task": "L1_003", "reader": "gpt-6-astra", "rep": 0, "family": "matmul", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.5027, "final": {"score": 1.5027, "status": "ok", "correct": true, "speedup": 1.5027, "speedup_lo95": 1.5006, "rid": "rk-final|ra-w1-0027|f4f2395e00030f05c14672e1", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5149, "code_sha": "f4f2395e00030f05c14672e1"}, "show_all": {"role": "S", "score": 1.5983, "final": {"score": 1.5983, "status": "ok", "correct": true, "speedup": 1.5983, "speedup_lo95": 1.5962, "rid": "rk-final|ra-w1-0027|bfa9f188c6b4c9b62a80cf3a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5952, "code_sha": "bfa9f188c6b4c9b62a80cf3a"}, "bare2": {"role": "P2_refine", "score": 1.9688, "final": {"score": 1.9688, "status": "ok", "correct": true, "speedup": 1.9688, "speedup_lo95": 1.9675, "rid": "rk-final|ra-w1-0027|fe2f8d53bd8cc8b67738e60a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.966, "code_sha": "fe2f8d53bd8cc8b67738e60a"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.5849, "final": {"score": 1.5849, "status": "ok", "correct": true, "speedup": 1.5849, "speedup_lo95": 1.5834, "rid": "rk-final|ra-w1-0027|6f8fa32a6733aa7343723d8c", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5774, "code_sha": "6f8fa32a6733aa7343723d8c"}, "audit_then_consult": {"role": "P1", "score": 1.5027, "final": {"score": 1.5027, "status": "ok", "correct": true, "speedup": 1.5027, "speedup_lo95": 1.5006, "rid": "rk-final|ra-w1-0027|f4f2395e00030f05c14672e1", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5149, "code_sha": "f4f2395e00030f05c14672e1"}, "prior_then_consult": {"role": "P2_prior", "score": 2.385, "final": {"score": 2.385, "status": "ok", "correct": true, "speedup": 2.385, "speedup_lo95": 2.38, "rid": "rk-final|ra-w1-0027|11abed47e1830c2815f7add5", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.3829, "code_sha": "11abed47e1830c2815f7add5"}, "consult_on_failure": {"role": "P2_refine", "score": 1.9688, "final": {"score": 1.9688, "status": "ok", "correct": true, "speedup": 1.9688, "speedup_lo95": 1.9675, "rid": "rk-final|ra-w1-0027|fe2f8d53bd8cc8b67738e60a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.966, "code_sha": "fe2f8d53bd8cc8b67738e60a"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0027/P1", "status": "ok", "correct": true, "speedup": 1.5149, "speedup_lo95": 1.5122, "ref_ms": 14.5693, "cand_ms": 9.61732, "score_pass": 1.5149, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1027, "tokens_out": 1284, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 242.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0027/P2_refine", "status": "ok", "correct": true, "speedup": 1.966, "speedup_lo95": 1.9621, "ref_ms": 14.50637, "cand_ms": 7.37872, "score_pass": 1.966, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1903, "tokens_out": 2429, "ctx_chars": 2820, "ctx_sha": "6dafa384125b8159", "wall_s": 247.7}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0027/P2_consult", "status": "ok", "correct": true, "speedup": 1.5774, "speedup_lo95": 1.5753, "ref_ms": 14.52191, "cand_ms": 9.20598, "score_pass": 1.5774, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10736, "tokens_out": 2853, "ctx_chars": 13509, "ctx_sha": "5ef4f15ce7ccb5a0", "wall_s": 306.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0027/P2_audit", "status": "ok", "correct": true, "speedup": 1.5066, "speedup_lo95": 1.5042, "ref_ms": 14.48637, "cand_ms": 9.61534, "score_pass": 1.5066, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10913, "tokens_out": 2794, "ctx_chars": 13821, "ctx_sha": "922e1b4b540b18fb", "wall_s": 319.7}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0027/P2_prior", "status": "ok", "correct": true, "speedup": 2.3829, "speedup_lo95": 2.3813, "ref_ms": 14.5402, "cand_ms": 6.10192, "score_pass": 2.3829, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2019, "tokens_out": 3833, "ctx_chars": 3448, "ctx_sha": "c9c1abd4ed29c065", "wall_s": 284.3}, "S": {"role": "S", "run_dir": "ra-w1-0027/S", "status": "ok", "correct": true, "speedup": 1.5952, "speedup_lo95": 1.5939, "ref_ms": 14.47882, "cand_ms": 9.07655, "score_pass": 1.5952, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8413, "tokens_out": 1655, "ctx_chars": 10384, "ctx_sha": "4573cb1c62d0cdf8", "wall_s": 641.1}}, "codes": {"P1": "f4f2395e00030f05c14672e1", "P2_refine": "fe2f8d53bd8cc8b67738e60a", "P2_consult": "6f8fa32a6733aa7343723d8c", "P2_audit": "213981ce4a9e15e1c0a65c47", "P2_prior": "11abed47e1830c2815f7add5", "S": "bfa9f188c6b4c9b62a80cf3a"}, "infra_any": false, "t_start": 1791534978.894318, "t_end": 1791535844.6911497}
{"block": "ra-w1-0023", "task": "L1_073", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": true, "arms": {"bare": {"role": "P1", "score": 0.3988, "final": {"score": 0.3988, "status": "ok", "correct": true, "speedup": 0.3988, "speedup_lo95": 0.3877, "rid": "rk-final|ra-w1-0023|3c83010f82c5b9daa6c91b37", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.4188, "code_sha": "3c83010f82c5b9daa6c91b37"}, "show_all": {"role": "S", "score": 0.1696, "final": {"score": 0.1696, "status": "ok", "correct": true, "speedup": 0.1696, "speedup_lo95": 0.1643, "rid": "rk-final|ra-w1-0023|dd3f9de5dd75bfd10e0205ae", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.2158, "code_sha": "dd3f9de5dd75bfd10e0205ae"}, "bare2": {"role": "P2_refine", "score": 1.8539, "final": {"score": 1.8539, "status": "ok", "correct": true, "speedup": 1.8539, "speedup_lo95": 1.8519, "rid": "rk-final|ra-w1-0023|9c189fb0cc595f3a0a1c3fbc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.8603, "code_sha": "9c189fb0cc595f3a0a1c3fbc"}, "attempt_then_consult": {"role": "P2_consult", "score": 0.8563, "final": {"score": 0.8563, "status": "ok", "correct": true, "speedup": 0.8563, "speedup_lo95": 0.7971, "rid": "rk-final|ra-w1-0023|cd8edd7ae3efbac132a9a3a6", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.8328, "code_sha": "cd8edd7ae3efbac132a9a3a6"}, "audit_then_consult": {"role": "P2_audit", "score": 0.9498, "final": {"score": 0.9498, "status": "ok", "correct": true, "speedup": 0.9498, "speedup_lo95": 0.8807, "rid": "rk-final|ra-w1-0023|64b794a7cae99f6d1e18cec2", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.8959, "code_sha": "64b794a7cae99f6d1e18cec2"}, "prior_then_consult": {"role": "P2_prior", "score": 4.7204, "final": {"score": 4.7204, "status": "ok", "correct": true, "speedup": 4.7204, "speedup_lo95": 4.6952, "rid": "rk-final|ra-w1-0023|72d356a3a589ea1b1bce6852", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.7694, "code_sha": "72d356a3a589ea1b1bce6852"}, "consult_on_failure": {"role": "P2_consult", "score": 0.8563, "final": {"score": 0.8563, "status": "ok", "correct": true, "speedup": 0.8563, "speedup_lo95": 0.7971, "rid": "rk-final|ra-w1-0023|cd8edd7ae3efbac132a9a3a6", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.8328, "code_sha": "cd8edd7ae3efbac132a9a3a6"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0023/P1", "status": "ok", "correct": true, "speedup": 0.4188, "speedup_lo95": 0.4022, "ref_ms": 20.31956, "cand_ms": 48.51675, "score_pass": 0.4188, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1293, "tokens_out": 2508, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 232.7}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0023/P2_refine", "status": "ok", "correct": true, "speedup": 1.8603, "speedup_lo95": 1.8559, "ref_ms": 19.80111, "cand_ms": 10.64431, "score_pass": 1.8603, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2902, "tokens_out": 2920, "ctx_chars": 6106, "ctx_sha": "5141448608801679", "wall_s": 217.0}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0023/P2_consult", "status": "ok", "correct": true, "speedup": 0.8328, "speedup_lo95": 0.7909, "ref_ms": 19.62225, "cand_ms": 23.5628, "score_pass": 0.8328, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12778, "tokens_out": 2229, "ctx_chars": 17586, "ctx_sha": "9114ec11ade217b0", "wall_s": 457.6}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0023/P2_audit", "status": "ok", "correct": true, "speedup": 0.8959, "speedup_lo95": 0.8886, "ref_ms": 19.5608, "cand_ms": 21.83345, "score_pass": 0.8959, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12917, "tokens_out": 2047, "ctx_chars": 17898, "ctx_sha": "ef31aa0941db001b", "wall_s": 607.8}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0023/P2_prior", "status": "ok", "correct": true, "speedup": 4.7694, "speedup_lo95": 4.6183, "ref_ms": 20.2452, "cand_ms": 4.24483, "score_pass": 4.7694, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3018, "tokens_out": 3213, "ctx_chars": 6734, "ctx_sha": "7d8fbb24c6d264e1", "wall_s": 517.7}, "S": {"role": "S", "run_dir": "ra-w1-0023/S", "status": "ok", "correct": true, "speedup": 0.2158, "speedup_lo95": 0.1874, "ref_ms": 24.52817, "cand_ms": 113.66907, "score_pass": 0.2158, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 17361, "tokens_out": 5298, "ctx_chars": 11175, "ctx_sha": "5e031feda794c122", "wall_s": 612.1}}, "codes": {"P1": "3c83010f82c5b9daa6c91b37", "P2_refine": "9c189fb0cc595f3a0a1c3fbc", "P2_consult": "cd8edd7ae3efbac132a9a3a6", "P2_audit": "64b794a7cae99f6d1e18cec2", "P2_prior": "72d356a3a589ea1b1bce6852", "S": "dd3f9de5dd75bfd10e0205ae"}, "infra_any": false, "t_start": 1791534758.8758328, "t_end": 1791535865.9154356}
{"block": "ra-w1-0026", "task": "L2_093", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 0.9606, "final": {"score": 0.9606, "status": "ok", "correct": true, "speedup": 0.9606, "speedup_lo95": 0.9599, "rid": "rk-final|ra-w1-0026|4a9d36752a0ee2c4e6474b42", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0059, "code_sha": "4a9d36752a0ee2c4e6474b42"}, "show_all": {"role": "S", "score": 1.131, "final": {"score": 1.131, "status": "ok", "correct": true, "speedup": 1.131, "speedup_lo95": 0.9617, "rid": "rk-final|ra-w1-0026|6c8de2c9a47c9fc73fc1d792", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9845, "code_sha": "6c8de2c9a47c9fc73fc1d792"}, "bare2": {"role": "P2_refine", "score": 3.7788, "final": {"score": 3.7788, "status": "ok", "correct": true, "speedup": 3.7788, "speedup_lo95": 3.7665, "rid": "rk-final|ra-w1-0026|703bfd2748dc37cc57e55246", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.7698, "code_sha": "703bfd2748dc37cc57e55246"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.0644, "final": {"score": 1.0644, "status": "ok", "correct": true, "speedup": 1.0644, "speedup_lo95": 1.0268, "rid": "rk-final|ra-w1-0026|f646f91192aaf2e4106ef0bf", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2143, "code_sha": "f646f91192aaf2e4106ef0bf"}, "audit_then_consult": {"role": "P2_audit", "score": 0.9504, "final": {"score": 0.9504, "status": "ok", "correct": true, "speedup": 0.9504, "speedup_lo95": 0.897, "rid": "rk-final|ra-w1-0026|ef0d88419bd4b9deb75eb079", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0129, "code_sha": "ef0d88419bd4b9deb75eb079"}, "prior_then_consult": {"role": "P1", "score": 0.9606, "final": {"score": 0.9606, "status": "ok", "correct": true, "speedup": 0.9606, "speedup_lo95": 0.9599, "rid": "rk-final|ra-w1-0026|4a9d36752a0ee2c4e6474b42", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0059, "code_sha": "4a9d36752a0ee2c4e6474b42"}, "consult_on_failure": {"role": "P2_refine", "score": 3.7788, "final": {"score": 3.7788, "status": "ok", "correct": true, "speedup": 3.7788, "speedup_lo95": 3.7665, "rid": "rk-final|ra-w1-0026|703bfd2748dc37cc57e55246", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.7698, "code_sha": "703bfd2748dc37cc57e55246"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0026/P1", "status": "ok", "correct": true, "speedup": 1.0059, "speedup_lo95": 0.9616, "ref_ms": 31.27815, "cand_ms": 31.09394, "score_pass": 1.0059, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1086, "tokens_out": 675, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 208.5}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0026/P2_refine", "status": "ok", "correct": true, "speedup": 3.7698, "speedup_lo95": 3.7636, "ref_ms": 31.26695, "cand_ms": 8.29406, "score_pass": 3.7698, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1557, "tokens_out": 2781, "ctx_chars": 1805, "ctx_sha": "4609540ce55431cb", "wall_s": 290.2}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0026/P2_consult", "status": "ok", "correct": true, "speedup": 1.2143, "speedup_lo95": 1.0262, "ref_ms": 31.20006, "cand_ms": 25.69387, "score_pass": 1.2143, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11294, "tokens_out": 3798, "ctx_chars": 13865, "ctx_sha": "5355ca1dd0ed87f1", "wall_s": 570.0}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0026/P2_audit", "status": "ok", "correct": true, "speedup": 1.0129, "speedup_lo95": 0.9037, "ref_ms": 31.42225, "cand_ms": 31.02231, "score_pass": 1.0129, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11315, "tokens_out": 3540, "ctx_chars": 14177, "ctx_sha": "c665561fbe1cf7d1", "wall_s": 358.5}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0026/P2_prior", "status": "ok", "correct": true, "speedup": 0.7527, "speedup_lo95": 0.6521, "ref_ms": 31.21772, "cand_ms": 41.47594, "score_pass": 0.7527, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1673, "tokens_out": 2397, "ctx_chars": 2433, "ctx_sha": "fd6bad46bf2bdf76", "wall_s": 249.1}, "S": {"role": "S", "run_dir": "ra-w1-0026/S", "status": "ok", "correct": true, "speedup": 0.9845, "speedup_lo95": 0.9695, "ref_ms": 31.54969, "cand_ms": 32.04631, "score_pass": 0.9845, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9162, "tokens_out": 773, "ctx_chars": 11755, "ctx_sha": "c9bcbb50b8ac09fd", "wall_s": 580.5}}, "codes": {"P1": "4a9d36752a0ee2c4e6474b42", "P2_refine": "703bfd2748dc37cc57e55246", "P2_consult": "f646f91192aaf2e4106ef0bf", "P2_audit": "ef0d88419bd4b9deb75eb079", "P2_prior": "98b12d074dcd4f304b5f7807", "S": "6c8de2c9a47c9fc73fc1d792"}, "infra_any": false, "t_start": 1791534923.7864704, "t_end": 1791535965.7795935}
{"block": "ra-w1-0029", "task": "L2_017", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.1798, "final": {"score": 1.1798, "status": "ok", "correct": true, "speedup": 1.1798, "speedup_lo95": 1.1757, "rid": "rk-final|ra-w1-0029|9324e2269e71d7afc424caf9", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0927, "code_sha": "9324e2269e71d7afc424caf9"}, "show_all": {"role": "S", "score": 1.1259, "final": {"score": 1.1259, "status": "ok", "correct": true, "speedup": 1.1259, "speedup_lo95": 1.0219, "rid": "rk-final|ra-w1-0029|8e106fe0601a1e02c428afa8", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0028, "code_sha": "8e106fe0601a1e02c428afa8"}, "bare2": {"role": "P2_refine", "score": 1.1585, "final": {"score": 1.1585, "status": "ok", "correct": true, "speedup": 1.1585, "speedup_lo95": 0.9348, "rid": "rk-final|ra-w1-0029|8b8dd189d0ba6cd5b4947184", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1643, "code_sha": "8b8dd189d0ba6cd5b4947184"}, "attempt_then_consult": {"role": "P2_consult", "score": 0.9456, "final": {"score": 0.9456, "status": "ok", "correct": true, "speedup": 0.9456, "speedup_lo95": 0.9257, "rid": "rk-final|ra-w1-0029|4ca9b78659de29da87beab86", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1483, "code_sha": "4ca9b78659de29da87beab86"}, "audit_then_consult": {"role": "P2_audit", "score": 1.0654, "final": {"score": 1.0654, "status": "ok", "correct": true, "speedup": 1.0654, "speedup_lo95": 0.8779, "rid": "rk-final|ra-w1-0029|fc4e9ca41375a54c33fa23f3", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1514, "code_sha": "fc4e9ca41375a54c33fa23f3"}, "prior_then_consult": {"role": "P2_prior", "score": 1.1322, "final": {"score": 1.1322, "status": "ok", "correct": true, "speedup": 1.1322, "speedup_lo95": 0.9092, "rid": "rk-final|ra-w1-0029|d451aa5457f6b059eb82225d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1246, "code_sha": "d451aa5457f6b059eb82225d"}, "consult_on_failure": {"role": "P2_refine", "score": 1.1585, "final": {"score": 1.1585, "status": "ok", "correct": true, "speedup": 1.1585, "speedup_lo95": 0.9348, "rid": "rk-final|ra-w1-0029|8b8dd189d0ba6cd5b4947184", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1643, "code_sha": "8b8dd189d0ba6cd5b4947184"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0029/P1", "status": "ok", "correct": true, "speedup": 1.0927, "speedup_lo95": 1.0283, "ref_ms": 18.39466, "cand_ms": 16.83388, "score_pass": 1.0927, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1014, "tokens_out": 900, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 207.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0029/P2_refine", "status": "ok", "correct": true, "speedup": 1.1643, "speedup_lo95": 1.1068, "ref_ms": 17.147, "cand_ms": 14.72704, "score_pass": 1.1643, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1557, "tokens_out": 2458, "ctx_chars": 2059, "ctx_sha": "30ea5d8c37873d2f", "wall_s": 197.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0029/P2_consult", "status": "ok", "correct": true, "speedup": 1.1483, "speedup_lo95": 1.0565, "ref_ms": 17.21135, "cand_ms": 14.98844, "score_pass": 1.1483, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10296, "tokens_out": 2998, "ctx_chars": 13496, "ctx_sha": "ece03d93126571d5", "wall_s": 228.7}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0029/P2_audit", "status": "ok", "correct": true, "speedup": 1.1514, "speedup_lo95": 1.0797, "ref_ms": 18.38803, "cand_ms": 15.97036, "score_pass": 1.1514, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10544, "tokens_out": 3175, "ctx_chars": 13808, "ctx_sha": "c5b47a8c2f244007", "wall_s": 333.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0029/P2_prior", "status": "ok", "correct": true, "speedup": 1.1246, "speedup_lo95": 0.9467, "ref_ms": 17.09103, "cand_ms": 15.19729, "score_pass": 1.1246, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1673, "tokens_out": 2636, "ctx_chars": 2687, "ctx_sha": "87ad6ea7bcd7ad3b", "wall_s": 183.3}, "S": {"role": "S", "run_dir": "ra-w1-0029/S", "status": "ok", "correct": true, "speedup": 1.0028, "speedup_lo95": 0.9645, "ref_ms": 17.07348, "cand_ms": 17.02567, "score_pass": 1.0028, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8690, "tokens_out": 1546, "ctx_chars": 11132, "ctx_sha": "4caa9b6e3a0949e1", "wall_s": 218.5}}, "codes": {"P1": "9324e2269e71d7afc424caf9", "P2_refine": "8b8dd189d0ba6cd5b4947184", "P2_consult": "4ca9b78659de29da87beab86", "P2_audit": "fc4e9ca41375a54c33fa23f3", "P2_prior": "d451aa5457f6b059eb82225d", "S": "8e106fe0601a1e02c428afa8"}, "infra_any": false, "t_start": 1791535221.7530117, "t_end": 1791536015.779362}
{"block": "ra-w1-0025", "task": "L2_001", "reader": "gpt-6-astra", "rep": 0, "family": "activation", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}, "show_all": {"role": "S", "score": 1.0843, "final": {"score": 1.0843, "status": "ok", "correct": true, "speedup": 1.0843, "speedup_lo95": 1.0752, "rid": "rk-final|ra-w1-0025|6a677cc83088af6dfdd40873", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0005, "code_sha": "6a677cc83088af6dfdd40873"}, "bare2": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}, "attempt_then_consult": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}, "audit_then_consult": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}, "prior_then_consult": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}, "consult_on_failure": {"role": "P1", "score": 1.1027, "final": {"score": 1.1027, "status": "ok", "correct": true, "speedup": 1.1027, "speedup_lo95": 1.0979, "rid": "rk-final|ra-w1-0025|a9d9ee1c17476ebe1d0dd43f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1047, "code_sha": "a9d9ee1c17476ebe1d0dd43f"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0025/P1", "status": "ok", "correct": true, "speedup": 1.1047, "speedup_lo95": 1.1011, "ref_ms": 16.06654, "cand_ms": 14.5441, "score_pass": 1.1047, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1008, "tokens_out": 868, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 193.0}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0025/P2_refine", "status": "ok", "correct": true, "speedup": 0.999, "speedup_lo95": 0.9726, "ref_ms": 16.3475, "cand_ms": 16.36305, "score_pass": 0.999, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1588, "tokens_out": 4907, "ctx_chars": 2065, "ctx_sha": "d4301071d31d4c31", "wall_s": 428.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0025/P2_consult", "status": "ok", "correct": true, "speedup": 1.0488, "speedup_lo95": 0.9937, "ref_ms": 15.76214, "cand_ms": 15.02826, "score_pass": 1.0488, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 5230, "tokens_out": 3348, "ctx_chars": 4352, "ctx_sha": "0b827384b9723c2e", "wall_s": 585.1}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0025/P2_audit", "status": "ok", "correct": true, "speedup": 1.0381, "speedup_lo95": 0.9306, "ref_ms": 15.73169, "cand_ms": 15.15412, "score_pass": 1.0381, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 4, "tokens_in": 15296, "tokens_out": 6258, "ctx_chars": 4664, "ctx_sha": "4a4c11e5425386ee", "wall_s": 730.8}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0025/P2_prior", "status": "ok", "correct": true, "speedup": 1.0826, "speedup_lo95": 0.9541, "ref_ms": 16.08166, "cand_ms": 14.85447, "score_pass": 1.0826, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1704, "tokens_out": 2979, "ctx_chars": 2693, "ctx_sha": "80dd9dc6de8f41fa", "wall_s": 377.5}, "S": {"role": "S", "run_dir": "ra-w1-0025/S", "status": "ok", "correct": true, "speedup": 1.0005, "speedup_lo95": 0.9883, "ref_ms": 16.01865, "cand_ms": 16.01068, "score_pass": 1.0005, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 3452, "tokens_out": 1174, "ctx_chars": 1982, "ctx_sha": "54e16caf9bef9eec", "wall_s": 240.5}}, "codes": {"P1": "a9d9ee1c17476ebe1d0dd43f", "P2_refine": "679269623057e691b41618b5", "P2_consult": "1f20e50bae50e4e293007e7a", "P2_audit": "e2d141f92cbcbfe6990dad39", "P2_prior": "d84e47a22856949985bec69f", "S": "6a677cc83088af6dfdd40873"}, "infra_any": false, "t_start": 1791534816.4785097, "t_end": 1791536075.1271305}
{"block": "ra-w1-0028", "task": "L3_025", "reader": "gpt-6-astra", "rep": 0, "family": "network", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.3842, "final": {"score": 1.3842, "status": "ok", "correct": true, "speedup": 1.3842, "speedup_lo95": 1.3824, "rid": "rk-final|ra-w1-0028|da8829185721b779a07555ca", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3778, "code_sha": "da8829185721b779a07555ca"}, "show_all": {"role": "S", "score": 1.3846, "final": {"score": 1.3846, "status": "ok", "correct": true, "speedup": 1.3846, "speedup_lo95": 1.3841, "rid": "rk-final|ra-w1-0028|c8e3a7c3f053d24c7567b74b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3833, "code_sha": "c8e3a7c3f053d24c7567b74b"}, "bare2": {"role": "P2_refine", "score": 1.4347, "final": {"score": 1.4347, "status": "ok", "correct": true, "speedup": 1.4347, "speedup_lo95": 1.4329, "rid": "rk-final|ra-w1-0028|07e60a11423f2097c8a7e25a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4745, "code_sha": "07e60a11423f2097c8a7e25a"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.6934, "final": {"score": 1.6934, "status": "ok", "correct": true, "speedup": 1.6934, "speedup_lo95": 1.6919, "rid": "rk-final|ra-w1-0028|4338df07dc3b5c7e5e36cdbc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6919, "code_sha": "4338df07dc3b5c7e5e36cdbc"}, "audit_then_consult": {"role": "P2_audit", "score": 1.4075, "final": {"score": 1.4075, "status": "ok", "correct": true, "speedup": 1.4075, "speedup_lo95": 1.4053, "rid": "rk-final|ra-w1-0028|2956c000eb997ed023a37ceb", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4823, "code_sha": "2956c000eb997ed023a37ceb"}, "prior_then_consult": {"role": "P2_prior", "score": 1.9078, "final": {"score": 1.9078, "status": "ok", "correct": true, "speedup": 1.9078, "speedup_lo95": 1.9071, "rid": "rk-final|ra-w1-0028|fd74d38c706b1f917dcdc0a9", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9071, "code_sha": "fd74d38c706b1f917dcdc0a9"}, "consult_on_failure": {"role": "P2_refine", "score": 1.4347, "final": {"score": 1.4347, "status": "ok", "correct": true, "speedup": 1.4347, "speedup_lo95": 1.4329, "rid": "rk-final|ra-w1-0028|07e60a11423f2097c8a7e25a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4745, "code_sha": "07e60a11423f2097c8a7e25a"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0028/P1", "status": "ok", "correct": true, "speedup": 1.3778, "speedup_lo95": 1.1769, "ref_ms": 20.89893, "cand_ms": 15.16818, "score_pass": 1.3778, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1575, "tokens_out": 3154, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 337.6}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0028/P2_refine", "status": "ok", "correct": true, "speedup": 1.4745, "speedup_lo95": 1.4706, "ref_ms": 20.81972, "cand_ms": 14.11989, "score_pass": 1.4745, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3915, "tokens_out": 3695, "ctx_chars": 7888, "ctx_sha": "783912cba6e6e38a", "wall_s": 196.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0028/P2_consult", "status": "ok", "correct": true, "speedup": 1.6919, "speedup_lo95": 1.6912, "ref_ms": 20.76118, "cand_ms": 12.27104, "score_pass": 1.6919, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 5628, "tokens_out": 3511, "ctx_chars": 14706, "ctx_sha": "9c528c468d2bc28a", "wall_s": 185.7}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0028/P2_audit", "status": "ok", "correct": true, "speedup": 1.4823, "speedup_lo95": 1.4787, "ref_ms": 20.73239, "cand_ms": 13.98686, "score_pass": 1.4823, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14374, "tokens_out": 6329, "ctx_chars": 15018, "ctx_sha": "24ab8eb4280bef1d", "wall_s": 246.4}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0028/P2_prior", "status": "ok", "correct": true, "speedup": 1.9071, "speedup_lo95": 1.8933, "ref_ms": 20.71635, "cand_ms": 10.86275, "score_pass": 1.9071, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 4031, "tokens_out": 4937, "ctx_chars": 8516, "ctx_sha": "efa3618fc1f44cdd", "wall_s": 361.6}, "S": {"role": "S", "run_dir": "ra-w1-0028/S", "status": "ok", "correct": true, "speedup": 1.3833, "speedup_lo95": 1.3822, "ref_ms": 20.69667, "cand_ms": 14.96232, "score_pass": 1.3833, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8589, "tokens_out": 4669, "ctx_chars": 6513, "ctx_sha": "1aa698d3a9589f25", "wall_s": 497.0}}, "codes": {"P1": "da8829185721b779a07555ca", "P2_refine": "07e60a11423f2097c8a7e25a", "P2_consult": "4338df07dc3b5c7e5e36cdbc", "P2_audit": "2956c000eb997ed023a37ceb", "P2_prior": "fd74d38c706b1f917dcdc0a9", "S": "c8e3a7c3f053d24c7567b74b"}, "infra_any": false, "t_start": 1791535125.9560332, "t_end": 1791536123.7058997}
{"block": "ra-w1-0030", "task": "L1_079", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.6343, "final": {"score": 1.6343, "status": "ok", "correct": true, "speedup": 1.6343, "speedup_lo95": 1.5065, "rid": "rk-final|ra-w1-0030|d072369b52cf8e74e04d5480", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5132, "code_sha": "d072369b52cf8e74e04d5480"}, "show_all": {"role": "S", "score": 0.6747, "final": {"score": 0.6747, "status": "ok", "correct": true, "speedup": 0.6747, "speedup_lo95": 0.6663, "rid": "rk-final|ra-w1-0030|24ac92dd510c33f9a29e5ade", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.5287, "code_sha": "24ac92dd510c33f9a29e5ade"}, "bare2": {"role": "P2_refine", "score": 3.3567, "final": {"score": 3.3567, "status": "ok", "correct": true, "speedup": 3.3567, "speedup_lo95": 3.3141, "rid": "rk-final|ra-w1-0030|2e701746b83f413c27d491b8", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.8029, "code_sha": "2e701746b83f413c27d491b8"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.5079, "final": {"score": 2.5079, "status": "ok", "correct": true, "speedup": 2.5079, "speedup_lo95": 2.3864, "rid": "rk-final|ra-w1-0030|468d47d3b715439059572d2a", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.3456, "code_sha": "468d47d3b715439059572d2a"}, "audit_then_consult": {"role": "P1", "score": 1.6343, "final": {"score": 1.6343, "status": "ok", "correct": true, "speedup": 1.6343, "speedup_lo95": 1.5065, "rid": "rk-final|ra-w1-0030|d072369b52cf8e74e04d5480", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.5132, "code_sha": "d072369b52cf8e74e04d5480"}, "prior_then_consult": {"role": "P2_prior", "score": 1.9675, "final": {"score": 1.9675, "status": "ok", "correct": true, "speedup": 1.9675, "speedup_lo95": 1.9636, "rid": "rk-final|ra-w1-0030|7be44d3fe2b3414f073d41cc", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.975, "code_sha": "7be44d3fe2b3414f073d41cc"}, "consult_on_failure": {"role": "P2_refine", "score": 3.3567, "final": {"score": 3.3567, "status": "ok", "correct": true, "speedup": 3.3567, "speedup_lo95": 3.3141, "rid": "rk-final|ra-w1-0030|2e701746b83f413c27d491b8", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.8029, "code_sha": "2e701746b83f413c27d491b8"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0030/P1", "status": "ok", "correct": true, "speedup": 1.5132, "speedup_lo95": 1.5122, "ref_ms": 8.58244, "cand_ms": 5.67176, "score_pass": 1.5132, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1257, "tokens_out": 2412, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 223.4}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0030/P2_refine", "status": "ok", "correct": true, "speedup": 2.8029, "speedup_lo95": 2.7987, "ref_ms": 8.47483, "cand_ms": 3.0236, "score_pass": 2.8029, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2523, "tokens_out": 3354, "ctx_chars": 4701, "ctx_sha": "15b74a1f3b9ef03f", "wall_s": 380.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0030/P2_consult", "status": "ok", "correct": true, "speedup": 2.3456, "speedup_lo95": 2.3403, "ref_ms": 8.61456, "cand_ms": 3.67267, "score_pass": 2.3456, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12746, "tokens_out": 4784, "ctx_chars": 16181, "ctx_sha": "2095700ef5c9590c", "wall_s": 502.8}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0030/P2_audit", "status": "ok", "correct": true, "speedup": 1.3535, "speedup_lo95": 1.3483, "ref_ms": 8.56472, "cand_ms": 6.32763, "score_pass": 1.3535, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12835, "tokens_out": 4512, "ctx_chars": 16493, "ctx_sha": "f1c4580e1464964c", "wall_s": 622.5}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0030/P2_prior", "status": "ok", "correct": true, "speedup": 1.975, "speedup_lo95": 1.9668, "ref_ms": 8.56055, "cand_ms": 4.33435, "score_pass": 1.975, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2639, "tokens_out": 2920, "ctx_chars": 5329, "ctx_sha": "189ca57780d74484", "wall_s": 251.2}, "S": {"role": "S", "run_dir": "ra-w1-0030/S", "status": "ok", "correct": true, "speedup": 0.5287, "speedup_lo95": 0.5278, "ref_ms": 10.31359, "cand_ms": 19.5092, "score_pass": 0.5287, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9640, "tokens_out": 3089, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 375.1}}, "codes": {"P1": "d072369b52cf8e74e04d5480", "P2_refine": "2e701746b83f413c27d491b8", "P2_consult": "468d47d3b715439059572d2a", "P2_audit": "6f5fd8b457ca8e44bce239a6", "P2_prior": "7be44d3fe2b3414f073d41cc", "S": "24ac92dd510c33f9a29e5ade"}, "infra_any": false, "t_start": 1791535355.355122, "t_end": 1791536449.6001112}
{"block": "ra-w1-0032", "task": "L1_075", "reader": "gpt-6-astra", "rep": 1, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 4.4417, "final": {"score": 4.4417, "status": "ok", "correct": true, "speedup": 4.4417, "speedup_lo95": 4.4314, "rid": "rk-final|ra-w1-0032|ec5fca007e73c7aaf3aff205", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.4342, "code_sha": "ec5fca007e73c7aaf3aff205"}, "show_all": {"role": "S", "score": 3.8138, "final": {"score": 3.8138, "status": "ok", "correct": true, "speedup": 3.8138, "speedup_lo95": 3.8052, "rid": "rk-final|ra-w1-0032|e2e0e6e3e5231433c8da3f63", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.8112, "code_sha": "e2e0e6e3e5231433c8da3f63"}, "bare2": {"role": "P2_refine", "score": 6.9903, "final": {"score": 6.9903, "status": "ok", "correct": true, "speedup": 6.9903, "speedup_lo95": 6.8766, "rid": "rk-final|ra-w1-0032|5b7df492148fdd0d091ca3d4", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.7994, "code_sha": "5b7df492148fdd0d091ca3d4"}, "attempt_then_consult": {"role": "P1", "score": 4.4417, "final": {"score": 4.4417, "status": "ok", "correct": true, "speedup": 4.4417, "speedup_lo95": 4.4314, "rid": "rk-final|ra-w1-0032|ec5fca007e73c7aaf3aff205", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.4342, "code_sha": "ec5fca007e73c7aaf3aff205"}, "audit_then_consult": {"role": "P2_audit", "score": 10.0, "final": {"score": 10.0, "status": "ok", "correct": true, "speedup": 10.6756, "speedup_lo95": 10.6592, "rid": "rk-final|ra-w1-0032|08b53cef8811d0c31084ac03", "infra": false, "tries": 1, "error": ""}, "score_pass": 10.0, "code_sha": "08b53cef8811d0c31084ac03"}, "prior_then_consult": {"role": "P2_prior", "score": 7.3457, "final": {"score": 7.3457, "status": "ok", "correct": true, "speedup": 7.3457, "speedup_lo95": 7.3323, "rid": "rk-final|ra-w1-0032|bebe450cfe38354518998131", "infra": false, "tries": 1, "error": ""}, "score_pass": 7.2929, "code_sha": "bebe450cfe38354518998131"}, "consult_on_failure": {"role": "P2_refine", "score": 6.9903, "final": {"score": 6.9903, "status": "ok", "correct": true, "speedup": 6.9903, "speedup_lo95": 6.8766, "rid": "rk-final|ra-w1-0032|5b7df492148fdd0d091ca3d4", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.7994, "code_sha": "5b7df492148fdd0d091ca3d4"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0032/P1", "status": "ok", "correct": true, "speedup": 4.4342, "speedup_lo95": 4.4296, "ref_ms": 9.94726, "cand_ms": 2.24329, "score_pass": 4.4342, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1353, "tokens_out": 3566, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 199.8}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0032/P2_refine", "status": "ok", "correct": true, "speedup": 6.7994, "speedup_lo95": 6.7826, "ref_ms": 9.81644, "cand_ms": 1.44372, "score_pass": 6.7994, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3059, "tokens_out": 2562, "ctx_chars": 5876, "ctx_sha": "a688393644c639b1", "wall_s": 200.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0032/P2_consult", "status": "ok", "correct": true, "speedup": 4.4249, "speedup_lo95": 4.4206, "ref_ms": 9.84951, "cand_ms": 2.22595, "score_pass": 4.4249, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13623, "tokens_out": 3705, "ctx_chars": 17356, "ctx_sha": "9df48058f1c9d6c6", "wall_s": 392.2}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0032/P2_audit", "status": "ok", "correct": true, "speedup": 10.6967, "speedup_lo95": 10.6776, "ref_ms": 9.765, "cand_ms": 0.9129, "score_pass": 10.0, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14308, "tokens_out": 4296, "ctx_chars": 17668, "ctx_sha": "7643a566a993d0fa", "wall_s": 314.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0032/P2_prior", "status": "ok", "correct": true, "speedup": 7.2929, "speedup_lo95": 7.2588, "ref_ms": 9.83895, "cand_ms": 1.34912, "score_pass": 7.2929, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3175, "tokens_out": 2951, "ctx_chars": 6504, "ctx_sha": "7737f13acee848a3", "wall_s": 270.3}, "S": {"role": "S", "run_dir": "ra-w1-0032/S", "status": "ok", "correct": true, "speedup": 3.8112, "speedup_lo95": 3.8036, "ref_ms": 9.87317, "cand_ms": 2.59059, "score_pass": 3.8112, "error": "", "infra": false, "infra_reruns": 0, "calls": 4, "tokens_in": 28593, "tokens_out": 9438, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 774.1}}, "codes": {"P1": "ec5fca007e73c7aaf3aff205", "P2_refine": "5b7df492148fdd0d091ca3d4", "P2_consult": "a96c73c7b500891cc1a1bae4", "P2_audit": "08b53cef8811d0c31084ac03", "P2_prior": "bebe450cfe38354518998131", "S": "e2e0e6e3e5231433c8da3f63"}, "infra_any": false, "t_start": 1791535507.9048126, "t_end": 1791536493.668146}
{"block": "ra-w1-0038", "task": "L1_098", "reader": "gpt-6-astra", "rep": 0, "family": "loss", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}, "show_all": {"role": "S", "score": 5.1753, "final": {"score": 5.1753, "status": "ok", "correct": true, "speedup": 5.1753, "speedup_lo95": 5.1615, "rid": "rk-final|ra-w1-0038|d3cf3f62a54aafd6a8937a3b", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.2736, "code_sha": "d3cf3f62a54aafd6a8937a3b"}, "bare2": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}, "attempt_then_consult": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}, "audit_then_consult": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}, "prior_then_consult": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}, "consult_on_failure": {"role": "P1", "score": 5.266, "final": {"score": 5.266, "status": "ok", "correct": true, "speedup": 5.266, "speedup_lo95": 5.2205, "rid": "rk-final|ra-w1-0038|a66ec5869bb94194507b8066", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.27, "code_sha": "a66ec5869bb94194507b8066"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0038/P1", "status": "ok", "correct": true, "speedup": 5.27, "speedup_lo95": 5.2412, "ref_ms": 8.63967, "cand_ms": 1.63942, "score_pass": 5.27, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 957, "tokens_out": 975, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 73.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0038/P2_refine", "status": "ok", "correct": true, "speedup": 5.1727, "speedup_lo95": 5.1658, "ref_ms": 8.64967, "cand_ms": 1.67217, "score_pass": 5.1727, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1593, "tokens_out": 1629, "ctx_chars": 2532, "ctx_sha": "a33d101b5a71206b", "wall_s": 56.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0038/P2_consult", "status": "ok", "correct": true, "speedup": 5.2175, "speedup_lo95": 5.2034, "ref_ms": 8.64348, "cand_ms": 1.65665, "score_pass": 5.2175, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 4836, "tokens_out": 2542, "ctx_chars": 4819, "ctx_sha": "fd663a87c1f1dce8", "wall_s": 193.6}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0038/P2_audit", "status": "ok", "correct": true, "speedup": 4.7914, "speedup_lo95": 4.788, "ref_ms": 8.62141, "cand_ms": 1.79934, "score_pass": 4.7914, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2162, "tokens_out": 1352, "ctx_chars": 5131, "ctx_sha": "02d162042086de93", "wall_s": 76.7}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0038/P2_prior", "status": "ok", "correct": true, "speedup": 5.1456, "speedup_lo95": 5.1361, "ref_ms": 8.62844, "cand_ms": 1.67685, "score_pass": 5.1456, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1709, "tokens_out": 1610, "ctx_chars": 3160, "ctx_sha": "13501ef847dfaaf9", "wall_s": 178.1}, "S": {"role": "S", "run_dir": "ra-w1-0038/S", "status": "ok", "correct": true, "speedup": 5.2736, "speedup_lo95": 5.2428, "ref_ms": 8.64387, "cand_ms": 1.63909, "score_pass": 5.2736, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 3378, "tokens_out": 1301, "ctx_chars": 1982, "ctx_sha": "54e16caf9bef9eec", "wall_s": 345.8}}, "codes": {"P1": "a66ec5869bb94194507b8066", "P2_refine": "3e1782c772d359ddd3b34423", "P2_consult": "24443c3ecd45d37c7547c938", "P2_audit": "8a1a148573a03939a4bbadde", "P2_prior": "d29595ba356e04d735be58b4", "S": "d3cf3f62a54aafd6a8937a3b"}, "infra_any": false, "t_start": 1791536075.127687, "t_end": 1791536494.1190104}
{"block": "ra-w1-0035", "task": "L2_007", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.3781, "final": {"score": 1.3781, "status": "ok", "correct": true, "speedup": 1.3781, "speedup_lo95": 1.3771, "rid": "rk-final|ra-w1-0035|1dda85abca67184dab91bec0", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3774, "code_sha": "1dda85abca67184dab91bec0"}, "show_all": {"role": "S", "score": 1.3798, "final": {"score": 1.3798, "status": "ok", "correct": true, "speedup": 1.3798, "speedup_lo95": 1.3788, "rid": "rk-final|ra-w1-0035|b8dcd0c8250b98fd23d1f419", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.378, "code_sha": "b8dcd0c8250b98fd23d1f419"}, "bare2": {"role": "P2_refine", "score": 2.9706, "final": {"score": 2.9706, "status": "ok", "correct": true, "speedup": 2.9706, "speedup_lo95": 2.965, "rid": "rk-final|ra-w1-0035|cc4a6ea8fc49597c3672f80e", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.9651, "code_sha": "cc4a6ea8fc49597c3672f80e"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.8804, "final": {"score": 2.8804, "status": "ok", "correct": true, "speedup": 2.8804, "speedup_lo95": 2.8779, "rid": "rk-final|ra-w1-0035|3e2290e598d4d3b8e6089d89", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.7753, "code_sha": "3e2290e598d4d3b8e6089d89"}, "audit_then_consult": {"role": "P2_audit", "score": 2.2256, "final": {"score": 2.2256, "status": "ok", "correct": true, "speedup": 2.2256, "speedup_lo95": 2.2185, "rid": "rk-final|ra-w1-0035|c2ecb2be8557a0aef30dc4b4", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.8481, "code_sha": "c2ecb2be8557a0aef30dc4b4"}, "prior_then_consult": {"role": "P2_prior", "score": 2.854, "final": {"score": 2.854, "status": "ok", "correct": true, "speedup": 2.854, "speedup_lo95": 2.8414, "rid": "rk-final|ra-w1-0035|ca47c86504e274809e6cd369", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.856, "code_sha": "ca47c86504e274809e6cd369"}, "consult_on_failure": {"role": "P2_refine", "score": 2.9706, "final": {"score": 2.9706, "status": "ok", "correct": true, "speedup": 2.9706, "speedup_lo95": 2.965, "rid": "rk-final|ra-w1-0035|cc4a6ea8fc49597c3672f80e", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.9651, "code_sha": "cc4a6ea8fc49597c3672f80e"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0035/P1", "status": "ok", "correct": true, "speedup": 1.3774, "speedup_lo95": 1.3768, "ref_ms": 19.96955, "cand_ms": 14.49784, "score_pass": 1.3774, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1082, "tokens_out": 954, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 194.4}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0035/P2_refine", "status": "ok", "correct": true, "speedup": 2.9651, "speedup_lo95": 2.9609, "ref_ms": 20.04819, "cand_ms": 6.76149, "score_pass": 2.9651, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1760, "tokens_out": 2517, "ctx_chars": 2261, "ctx_sha": "29c76b18a656771b", "wall_s": 163.2}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0035/P2_consult", "status": "ok", "correct": true, "speedup": 2.7753, "speedup_lo95": 2.5994, "ref_ms": 20.04009, "cand_ms": 7.221, "score_pass": 2.7753, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10431, "tokens_out": 2914, "ctx_chars": 13074, "ctx_sha": "1b944f64d7f4a940", "wall_s": 337.8}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0035/P2_audit", "status": "ok", "correct": true, "speedup": 2.8481, "speedup_lo95": 2.8464, "ref_ms": 19.9888, "cand_ms": 7.01833, "score_pass": 2.8481, "error": "", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 18040, "tokens_out": 4711, "ctx_chars": 13386, "ctx_sha": "952c8171c25660c0", "wall_s": 275.7}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0035/P2_prior", "status": "ok", "correct": true, "speedup": 2.856, "speedup_lo95": 2.844, "ref_ms": 20.12852, "cand_ms": 7.04787, "score_pass": 2.856, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1876, "tokens_out": 2843, "ctx_chars": 2889, "ctx_sha": "1bfa4b69ebdbe0ef", "wall_s": 240.0}, "S": {"role": "S", "run_dir": "ra-w1-0035/S", "status": "ok", "correct": true, "speedup": 1.378, "speedup_lo95": 1.377, "ref_ms": 20.02667, "cand_ms": 14.53341, "score_pass": 1.378, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8482, "tokens_out": 1282, "ctx_chars": 10508, "ctx_sha": "4fa7857ea4c0dcf4", "wall_s": 310.0}}, "codes": {"P1": "1dda85abca67184dab91bec0", "P2_refine": "cc4a6ea8fc49597c3672f80e", "P2_consult": "3e2290e598d4d3b8e6089d89", "P2_audit": "c2ecb2be8557a0aef30dc4b4", "P2_prior": "ca47c86504e274809e6cd369", "S": "b8dcd0c8250b98fd23d1f419"}, "infra_any": false, "t_start": 1791535865.916317, "t_end": 1791536603.4345722}
{"block": "ra-w1-0036", "task": "L2_017", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": true, "arms": {"bare": {"role": "P1", "score": 0.9756, "final": {"score": 0.9756, "status": "ok", "correct": true, "speedup": 0.9756, "speedup_lo95": 0.9671, "rid": "rk-final|ra-w1-0036|a80578c8e1c09591ba0eb7d0", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9704, "code_sha": "a80578c8e1c09591ba0eb7d0"}, "show_all": {"role": "S", "score": 1.0673, "final": {"score": 1.0673, "status": "ok", "correct": true, "speedup": 1.0673, "speedup_lo95": 1.0412, "rid": "rk-final|ra-w1-0036|07f92cfea3fe08c73cb85ac1", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1789, "code_sha": "07f92cfea3fe08c73cb85ac1"}, "bare2": {"role": "P1", "score": 0.9756, "final": {"score": 0.9756, "status": "ok", "correct": true, "speedup": 0.9756, "speedup_lo95": 0.9671, "rid": "rk-final|ra-w1-0036|a80578c8e1c09591ba0eb7d0", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9704, "code_sha": "a80578c8e1c09591ba0eb7d0"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.183, "final": {"score": 1.183, "status": "ok", "correct": true, "speedup": 1.183, "speedup_lo95": 1.0102, "rid": "rk-final|ra-w1-0036|45dbc1e1d0042d79dd967125", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0693, "code_sha": "45dbc1e1d0042d79dd967125"}, "audit_then_consult": {"role": "P2_audit", "score": 1.0188, "final": {"score": 1.0188, "status": "ok", "correct": true, "speedup": 1.0188, "speedup_lo95": 1.0158, "rid": "rk-final|ra-w1-0036|e4850076acbcc43d5177672c", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.198, "code_sha": "e4850076acbcc43d5177672c"}, "prior_then_consult": {"role": "P1", "score": 0.9756, "final": {"score": 0.9756, "status": "ok", "correct": true, "speedup": 0.9756, "speedup_lo95": 0.9671, "rid": "rk-final|ra-w1-0036|a80578c8e1c09591ba0eb7d0", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.9704, "code_sha": "a80578c8e1c09591ba0eb7d0"}, "consult_on_failure": {"role": "P2_consult", "score": 1.183, "final": {"score": 1.183, "status": "ok", "correct": true, "speedup": 1.183, "speedup_lo95": 1.0102, "rid": "rk-final|ra-w1-0036|45dbc1e1d0042d79dd967125", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0693, "code_sha": "45dbc1e1d0042d79dd967125"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0036/P1", "status": "ok", "correct": true, "speedup": 0.9704, "speedup_lo95": 0.9578, "ref_ms": 16.94501, "cand_ms": 17.46209, "score_pass": 0.9704, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1014, "tokens_out": 1256, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 177.4}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0036/P2_refine", "status": "ok", "correct": true, "speedup": 0.9594, "speedup_lo95": 0.9421, "ref_ms": 17.55349, "cand_ms": 18.29614, "score_pass": 0.9594, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1930, "tokens_out": 4529, "ctx_chars": 3274, "ctx_sha": "5e86869ffca21531", "wall_s": 374.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0036/P2_consult", "status": "ok", "correct": true, "speedup": 1.0693, "speedup_lo95": 1.0206, "ref_ms": 17.99235, "cand_ms": 16.82611, "score_pass": 1.0693, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10970, "tokens_out": 2502, "ctx_chars": 14711, "ctx_sha": "27334964a15dfeba", "wall_s": 219.9}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0036/P2_audit", "status": "ok", "correct": true, "speedup": 1.198, "speedup_lo95": 1.0308, "ref_ms": 17.35339, "cand_ms": 14.48562, "score_pass": 1.198, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11206, "tokens_out": 2804, "ctx_chars": 15023, "ctx_sha": "d3a4e32ccd7a1447", "wall_s": 246.7}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0036/P2_prior", "status": "ok", "correct": true, "speedup": 0.847, "speedup_lo95": 0.8284, "ref_ms": 18.57525, "cand_ms": 21.92996, "score_pass": 0.847, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2046, "tokens_out": 3832, "ctx_chars": 3902, "ctx_sha": "c33c5bc77f217446", "wall_s": 258.7}, "S": {"role": "S", "run_dir": "ra-w1-0036/S", "status": "ok", "correct": true, "speedup": 1.1789, "speedup_lo95": 1.1739, "ref_ms": 16.9484, "cand_ms": 14.37612, "score_pass": 1.1789, "error": "", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 14389, "tokens_out": 3118, "ctx_chars": 11132, "ctx_sha": "4caa9b6e3a0949e1", "wall_s": 503.8}}, "codes": {"P1": "a80578c8e1c09591ba0eb7d0", "P2_refine": "033c9c1370980cc5658cb511", "P2_consult": "45dbc1e1d0042d79dd967125", "P2_audit": "e4850076acbcc43d5177672c", "P2_prior": "62c41231c6b46bd905132235", "S": "07f92cfea3fe08c73cb85ac1"}, "infra_any": false, "t_start": 1791535965.7804844, "t_end": 1791536736.85036}
{"block": "ra-w1-0037", "task": "L2_007", "reader": "gpt-6-astra", "rep": 1, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.125, "final": {"score": 1.125, "status": "ok", "correct": true, "speedup": 1.125, "speedup_lo95": 1.1225, "rid": "rk-final|ra-w1-0037|7d1266df00a9f03940f30f73", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3802, "code_sha": "7d1266df00a9f03940f30f73"}, "show_all": {"role": "S", "score": 1.3788, "final": {"score": 1.3788, "status": "ok", "correct": true, "speedup": 1.3788, "speedup_lo95": 1.378, "rid": "rk-final|ra-w1-0037|0efb77c35703b448fbd4f195", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3788, "code_sha": "0efb77c35703b448fbd4f195"}, "bare2": {"role": "P2_refine", "score": 2.1875, "final": {"score": 2.1875, "status": "ok", "correct": true, "speedup": 2.1875, "speedup_lo95": 2.1095, "rid": "rk-final|ra-w1-0037|bd99a1bb5556dac93acfe9f7", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.6516, "code_sha": "bd99a1bb5556dac93acfe9f7"}, "attempt_then_consult": {"role": "P2_consult", "score": 2.6172, "final": {"score": 2.6172, "status": "ok", "correct": true, "speedup": 2.6172, "speedup_lo95": 2.61, "rid": "rk-final|ra-w1-0037|fc89b414d9eef1435037b7d8", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.6304, "code_sha": "fc89b414d9eef1435037b7d8"}, "audit_then_consult": {"role": "P2_audit", "score": 2.5099, "final": {"score": 2.5099, "status": "ok", "correct": true, "speedup": 2.5099, "speedup_lo95": 2.471, "rid": "rk-final|ra-w1-0037|1e9b6a03857c3c0328ae4bb3", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9869, "code_sha": "1e9b6a03857c3c0328ae4bb3"}, "prior_then_consult": {"role": "P2_prior", "score": 2.7731, "final": {"score": 2.7731, "status": "ok", "correct": true, "speedup": 2.7731, "speedup_lo95": 2.4417, "rid": "rk-final|ra-w1-0037|a723a4abb7295f2990d24333", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.9326, "code_sha": "a723a4abb7295f2990d24333"}, "consult_on_failure": {"role": "P2_refine", "score": 2.1875, "final": {"score": 2.1875, "status": "ok", "correct": true, "speedup": 2.1875, "speedup_lo95": 2.1095, "rid": "rk-final|ra-w1-0037|bd99a1bb5556dac93acfe9f7", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.6516, "code_sha": "bd99a1bb5556dac93acfe9f7"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0037/P1", "status": "ok", "correct": true, "speedup": 1.3802, "speedup_lo95": 1.3769, "ref_ms": 20.08956, "cand_ms": 14.55597, "score_pass": 1.3802, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1082, "tokens_out": 993, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 147.7}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0037/P2_refine", "status": "ok", "correct": true, "speedup": 2.6516, "speedup_lo95": 2.6036, "ref_ms": 20.07837, "cand_ms": 7.57226, "score_pass": 2.6516, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1785, "tokens_out": 2127, "ctx_chars": 2405, "ctx_sha": "88949f80665bbf4c", "wall_s": 215.9}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0037/P2_consult", "status": "ok", "correct": true, "speedup": 2.6304, "speedup_lo95": 2.6284, "ref_ms": 20.05381, "cand_ms": 7.62383, "score_pass": 2.6304, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10615, "tokens_out": 3696, "ctx_chars": 13218, "ctx_sha": "d0b5a6d0a165040d", "wall_s": 278.2}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0037/P2_audit", "status": "ok", "correct": true, "speedup": 1.9869, "speedup_lo95": 1.9797, "ref_ms": 20.11834, "cand_ms": 10.12551, "score_pass": 1.9869, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10732, "tokens_out": 3300, "ctx_chars": 13530, "ctx_sha": "5b19e15bb2aec902", "wall_s": 349.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0037/P2_prior", "status": "ok", "correct": true, "speedup": 2.9326, "speedup_lo95": 2.4527, "ref_ms": 20.08665, "cand_ms": 6.84939, "score_pass": 2.9326, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1901, "tokens_out": 2478, "ctx_chars": 3033, "ctx_sha": "602ab26558b13177", "wall_s": 228.9}, "S": {"role": "S", "run_dir": "ra-w1-0037/S", "status": "ok", "correct": true, "speedup": 1.3788, "speedup_lo95": 1.3777, "ref_ms": 20.00382, "cand_ms": 14.50801, "score_pass": 1.3788, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8568, "tokens_out": 1470, "ctx_chars": 10508, "ctx_sha": "4fa7857ea4c0dcf4", "wall_s": 225.5}}, "codes": {"P1": "7d1266df00a9f03940f30f73", "P2_refine": "bd99a1bb5556dac93acfe9f7", "P2_consult": "fc89b414d9eef1435037b7d8", "P2_audit": "1e9b6a03857c3c0328ae4bb3", "P2_prior": "a723a4abb7295f2990d24333", "S": "0efb77c35703b448fbd4f195"}, "infra_any": false, "t_start": 1791536015.7799037, "t_end": 1791536746.433759}
{"block": "ra-w1-0033", "task": "L3_013", "reader": "gpt-6-astra", "rep": 0, "family": "network", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 4.3407, "final": {"score": 4.3407, "status": "ok", "correct": true, "speedup": 4.3407, "speedup_lo95": 4.2986, "rid": "rk-final|ra-w1-0033|e962ac47d6cb9d3b0d1acbdd", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.4398, "code_sha": "e962ac47d6cb9d3b0d1acbdd"}, "show_all": {"role": "S", "score": 1.6722, "final": {"score": 1.6722, "status": "ok", "correct": true, "speedup": 1.6722, "speedup_lo95": 1.668, "rid": "rk-final|ra-w1-0033|d7fc41e1d91b552ad2403423", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6694, "code_sha": "d7fc41e1d91b552ad2403423"}, "bare2": {"role": "P2_refine", "score": 6.3127, "final": {"score": 6.3127, "status": "ok", "correct": true, "speedup": 6.3127, "speedup_lo95": 6.2921, "rid": "rk-final|ra-w1-0033|73f055a320060c9fdf44d498", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.3657, "code_sha": "73f055a320060c9fdf44d498"}, "attempt_then_consult": {"role": "P2_consult", "score": 6.755, "final": {"score": 6.755, "status": "ok", "correct": true, "speedup": 6.755, "speedup_lo95": 6.7065, "rid": "rk-final|ra-w1-0033|42f207be412e965250b2cae8", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.6971, "code_sha": "42f207be412e965250b2cae8"}, "audit_then_consult": {"role": "P2_audit", "score": 6.3133, "final": {"score": 6.3133, "status": "ok", "correct": true, "speedup": 6.3133, "speedup_lo95": 6.2532, "rid": "rk-final|ra-w1-0033|966fc1b29c20d18adb63d110", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.3276, "code_sha": "966fc1b29c20d18adb63d110"}, "prior_then_consult": {"role": "P2_prior", "score": 6.6863, "final": {"score": 6.6863, "status": "ok", "correct": true, "speedup": 6.6863, "speedup_lo95": 6.6681, "rid": "rk-final|ra-w1-0033|11fa5ca4331374a9e7f12f0f", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.6658, "code_sha": "11fa5ca4331374a9e7f12f0f"}, "consult_on_failure": {"role": "P2_refine", "score": 6.3127, "final": {"score": 6.3127, "status": "ok", "correct": true, "speedup": 6.3127, "speedup_lo95": 6.2921, "rid": "rk-final|ra-w1-0033|73f055a320060c9fdf44d498", "infra": false, "tries": 1, "error": ""}, "score_pass": 6.3657, "code_sha": "73f055a320060c9fdf44d498"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0033/P1", "status": "ok", "correct": true, "speedup": 4.4398, "speedup_lo95": 4.4325, "ref_ms": 16.75838, "cand_ms": 3.77457, "score_pass": 4.4398, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1057, "tokens_out": 3331, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 233.4}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0033/P2_refine", "status": "ok", "correct": true, "speedup": 6.3657, "speedup_lo95": 6.3422, "ref_ms": 16.73146, "cand_ms": 2.62838, "score_pass": 6.3657, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2944, "tokens_out": 2899, "ctx_chars": 6601, "ctx_sha": "a3684194c324ee8f", "wall_s": 137.7}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0033/P2_consult", "status": "ok", "correct": true, "speedup": 6.6971, "speedup_lo95": 6.6875, "ref_ms": 16.60325, "cand_ms": 2.47917, "score_pass": 6.6971, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11259, "tokens_out": 4367, "ctx_chars": 13419, "ctx_sha": "5ca33d1f77dd1738", "wall_s": 245.4}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0033/P2_audit", "status": "ok", "correct": true, "speedup": 6.3276, "speedup_lo95": 6.2753, "ref_ms": 16.90553, "cand_ms": 2.67169, "score_pass": 6.3276, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11431, "tokens_out": 4896, "ctx_chars": 13731, "ctx_sha": "fca18e452c25f968", "wall_s": 536.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0033/P2_prior", "status": "ok", "correct": true, "speedup": 6.6658, "speedup_lo95": 6.6424, "ref_ms": 16.72187, "cand_ms": 2.5086, "score_pass": 6.6658, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3060, "tokens_out": 3085, "ctx_chars": 7229, "ctx_sha": "d817cdba95ef15ba", "wall_s": 137.3}, "S": {"role": "S", "run_dir": "ra-w1-0033/S", "status": "ok", "correct": true, "speedup": 1.6694, "speedup_lo95": 1.6658, "ref_ms": 16.66734, "cand_ms": 9.98382, "score_pass": 1.6694, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 6030, "tokens_out": 1720, "ctx_chars": 6513, "ctx_sha": "1aa698d3a9589f25", "wall_s": 331.7}}, "codes": {"P1": "e962ac47d6cb9d3b0d1acbdd", "P2_refine": "73f055a320060c9fdf44d498", "P2_consult": "42f207be412e965250b2cae8", "P2_audit": "966fc1b29c20d18adb63d110", "P2_prior": "11fa5ca4331374a9e7f12f0f", "S": "d7fc41e1d91b552ad2403423"}, "infra_any": false, "t_start": 1791535753.5121799, "t_end": 1791536763.6601942}
{"block": "ra-w1-0031", "task": "L1_075", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 3.1402, "final": {"score": 3.1402, "status": "ok", "correct": true, "speedup": 3.1402, "speedup_lo95": 3.1335, "rid": "rk-final|ra-w1-0031|1860e7d9fe311760b52fac2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.1534, "code_sha": "1860e7d9fe311760b52fac2f"}, "show_all": {"role": "S", "score": 2.5198, "final": {"score": 2.5198, "status": "ok", "correct": true, "speedup": 2.5198, "speedup_lo95": 2.5098, "rid": "rk-final|ra-w1-0031|0f8ed5f9785c6d5a49fcd41a", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.4952, "code_sha": "0f8ed5f9785c6d5a49fcd41a"}, "bare2": {"role": "P2_refine", "score": 7.1647, "final": {"score": 7.1647, "status": "ok", "correct": true, "speedup": 7.1647, "speedup_lo95": 7.0929, "rid": "rk-final|ra-w1-0031|8d54cca52da7f4239417f2cf", "infra": false, "tries": 1, "error": ""}, "score_pass": 7.12, "code_sha": "8d54cca52da7f4239417f2cf"}, "attempt_then_consult": {"role": "P2_consult", "score": 5.1671, "final": {"score": 5.1671, "status": "ok", "correct": true, "speedup": 5.1671, "speedup_lo95": 5.1611, "rid": "rk-final|ra-w1-0031|3276e13e9561c5aa12a02967", "infra": false, "tries": 1, "error": ""}, "score_pass": 5.1881, "code_sha": "3276e13e9561c5aa12a02967"}, "audit_then_consult": {"role": "P1", "score": 3.1402, "final": {"score": 3.1402, "status": "ok", "correct": true, "speedup": 3.1402, "speedup_lo95": 3.1335, "rid": "rk-final|ra-w1-0031|1860e7d9fe311760b52fac2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 3.1534, "code_sha": "1860e7d9fe311760b52fac2f"}, "prior_then_consult": {"role": "P2_prior", "score": 4.215, "final": {"score": 4.215, "status": "ok", "correct": true, "speedup": 4.215, "speedup_lo95": 4.2112, "rid": "rk-final|ra-w1-0031|9b10bd52b0fd9923c56170a2", "infra": false, "tries": 1, "error": ""}, "score_pass": 4.2385, "code_sha": "9b10bd52b0fd9923c56170a2"}, "consult_on_failure": {"role": "P2_refine", "score": 7.1647, "final": {"score": 7.1647, "status": "ok", "correct": true, "speedup": 7.1647, "speedup_lo95": 7.0929, "rid": "rk-final|ra-w1-0031|8d54cca52da7f4239417f2cf", "infra": false, "tries": 1, "error": ""}, "score_pass": 7.12, "code_sha": "8d54cca52da7f4239417f2cf"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0031/P1", "status": "ok", "correct": true, "speedup": 3.1534, "speedup_lo95": 3.1491, "ref_ms": 9.91681, "cand_ms": 3.14481, "score_pass": 3.1534, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1353, "tokens_out": 4041, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 429.0}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0031/P2_refine", "status": "ok", "correct": true, "speedup": 7.12, "speedup_lo95": 7.1077, "ref_ms": 9.76181, "cand_ms": 1.37103, "score_pass": 7.12, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3432, "tokens_out": 4672, "ctx_chars": 7019, "ctx_sha": "3b6e34e0c499743b", "wall_s": 224.9}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0031/P2_consult", "status": "ok", "correct": true, "speedup": 5.1881, "speedup_lo95": 5.1764, "ref_ms": 9.82876, "cand_ms": 1.8945, "score_pass": 5.1881, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 15173, "tokens_out": 6006, "ctx_chars": 18499, "ctx_sha": "e95264e980f0c780", "wall_s": 652.5}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0031/P2_audit", "status": "ok", "correct": true, "speedup": 2.6311, "speedup_lo95": 2.6289, "ref_ms": 9.83789, "cand_ms": 3.73903, "score_pass": 2.6311, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 15419, "tokens_out": 5555, "ctx_chars": 18811, "ctx_sha": "34c2194593c43573", "wall_s": 288.4}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0031/P2_prior", "status": "ok", "correct": true, "speedup": 4.2385, "speedup_lo95": 4.2318, "ref_ms": 9.92889, "cand_ms": 2.34257, "score_pass": 4.2385, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 3548, "tokens_out": 3318, "ctx_chars": 7647, "ctx_sha": "bbf8d8cf2d0d147b", "wall_s": 178.2}, "S": {"role": "S", "run_dir": "ra-w1-0031/S", "status": "ok", "correct": true, "speedup": 2.4952, "speedup_lo95": 2.4895, "ref_ms": 9.89183, "cand_ms": 3.96432, "score_pass": 2.4952, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 10624, "tokens_out": 4582, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 551.4}}, "codes": {"P1": "1860e7d9fe311760b52fac2f", "P2_refine": "8d54cca52da7f4239417f2cf", "P2_consult": "3276e13e9561c5aa12a02967", "P2_audit": "09ba48217284fb4a7ba696cf", "P2_prior": "9b10bd52b0fd9923c56170a2", "S": "0f8ed5f9785c6d5a49fcd41a"}, "infra_any": false, "t_start": 1791535507.4828916, "t_end": 1791536788.3320153}
{"block": "ra-w1-0041", "task": "L2_010", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}, "show_all": {"role": "S", "score": 1.0077, "final": {"score": 1.0077, "status": "ok", "correct": true, "speedup": 1.0077, "speedup_lo95": 1.0069, "rid": "rk-final|ra-w1-0041|2ba8768bfbdcf62f3bfb4a6d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.0014, "code_sha": "2ba8768bfbdcf62f3bfb4a6d"}, "bare2": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}, "attempt_then_consult": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}, "audit_then_consult": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}, "prior_then_consult": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}, "consult_on_failure": {"role": "P1", "score": 1.2883, "final": {"score": 1.2883, "status": "ok", "correct": true, "speedup": 1.2883, "speedup_lo95": 1.2875, "rid": "rk-final|ra-w1-0041|0496137f1e35f5f014efca2f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.2853, "code_sha": "0496137f1e35f5f014efca2f"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0041/P1", "status": "ok", "correct": true, "speedup": 1.2853, "speedup_lo95": 1.2848, "ref_ms": 25.4031, "cand_ms": 19.76476, "score_pass": 1.2853, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1165, "tokens_out": 2009, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 157.0}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0041/P2_refine", "status": "ok", "correct": true, "speedup": 0.7845, "speedup_lo95": 0.7842, "ref_ms": 25.41374, "cand_ms": 32.39485, "score_pass": 0.7845, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2467, "tokens_out": 4005, "ctx_chars": 4660, "ctx_sha": "c7863f6d2d1a164b", "wall_s": 142.1}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0041/P2_consult", "status": "ok", "correct": true, "speedup": 1.1101, "speedup_lo95": 1.1099, "ref_ms": 25.33598, "cand_ms": 22.82314, "score_pass": 1.1101, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14572, "tokens_out": 3630, "ctx_chars": 18672, "ctx_sha": "12c173a5885035a4", "wall_s": 162.9}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0041/P2_audit", "status": "ok", "correct": true, "speedup": 1.132, "speedup_lo95": 1.1116, "ref_ms": 25.42347, "cand_ms": 22.45862, "score_pass": 1.132, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 14812, "tokens_out": 3690, "ctx_chars": 18984, "ctx_sha": "6a81f1a02596b0e4", "wall_s": 208.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0041/P2_prior", "status": "ok", "correct": true, "speedup": 0.8046, "speedup_lo95": 0.8032, "ref_ms": 25.4261, "cand_ms": 31.60147, "score_pass": 0.8046, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2583, "tokens_out": 4114, "ctx_chars": 5288, "ctx_sha": "788a0c18022c098e", "wall_s": 121.5}, "S": {"role": "S", "run_dir": "ra-w1-0041/S", "status": "ok", "correct": true, "speedup": 1.0014, "speedup_lo95": 1.0012, "ref_ms": 25.44983, "cand_ms": 25.41342, "score_pass": 1.0014, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 11511, "tokens_out": 2679, "ctx_chars": 13707, "ctx_sha": "c2fd2e75f0a9c4ac", "wall_s": 234.3}}, "codes": {"P1": "0496137f1e35f5f014efca2f", "P2_refine": "f803a1ec96169bb4dc73365c", "P2_consult": "eb60b2774763c86f1b3f706f", "P2_audit": "8d1c04f45b2c9de1cc52acf1", "P2_prior": "6ede8793f6e562dda5ea6f10", "S": "2ba8768bfbdcf62f3bfb4a6d"}, "infra_any": false, "t_start": 1791536493.6686113, "t_end": 1791536873.1814263}
{"block": "ra-w1-0039", "task": "L1_054", "reader": "gpt-6-astra", "rep": 0, "family": "conv", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.6372, "final": {"score": 1.6372, "status": "ok", "correct": true, "speedup": 1.6372, "speedup_lo95": 1.6143, "rid": "rk-final|ra-w1-0039|66043f12083d46afadd02338", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7433, "code_sha": "66043f12083d46afadd02338"}, "show_all": {"role": "S", "score": 1.3659, "final": {"score": 1.3659, "status": "ok", "correct": true, "speedup": 1.3659, "speedup_lo95": 1.3647, "rid": "rk-final|ra-w1-0039|560dbc39a65f32dda0d2d26a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.3526, "code_sha": "560dbc39a65f32dda0d2d26a"}, "bare2": {"role": "P2_refine", "score": 2.701, "final": {"score": 2.701, "status": "ok", "correct": true, "speedup": 2.701, "speedup_lo95": 2.6975, "rid": "rk-final|ra-w1-0039|56a3e5139aab0c91c7b2ac96", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2455, "code_sha": "56a3e5139aab0c91c7b2ac96"}, "attempt_then_consult": {"role": "P1", "score": 1.6372, "final": {"score": 1.6372, "status": "ok", "correct": true, "speedup": 1.6372, "speedup_lo95": 1.6143, "rid": "rk-final|ra-w1-0039|66043f12083d46afadd02338", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7433, "code_sha": "66043f12083d46afadd02338"}, "audit_then_consult": {"role": "P2_audit", "score": 1.6553, "final": {"score": 1.6553, "status": "ok", "correct": true, "speedup": 1.6553, "speedup_lo95": 1.6491, "rid": "rk-final|ra-w1-0039|c68778aa48e7479286fc034a", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.851, "code_sha": "c68778aa48e7479286fc034a"}, "prior_then_consult": {"role": "P1", "score": 1.6372, "final": {"score": 1.6372, "status": "ok", "correct": true, "speedup": 1.6372, "speedup_lo95": 1.6143, "rid": "rk-final|ra-w1-0039|66043f12083d46afadd02338", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.7433, "code_sha": "66043f12083d46afadd02338"}, "consult_on_failure": {"role": "P2_refine", "score": 2.701, "final": {"score": 2.701, "status": "ok", "correct": true, "speedup": 2.701, "speedup_lo95": 2.6975, "rid": "rk-final|ra-w1-0039|56a3e5139aab0c91c7b2ac96", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.2455, "code_sha": "56a3e5139aab0c91c7b2ac96"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0039/P1", "status": "ok", "correct": true, "speedup": 1.7433, "speedup_lo95": 1.6281, "ref_ms": 5.13142, "cand_ms": 2.94352, "score_pass": 1.7433, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1278, "tokens_out": 2291, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 195.9}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0039/P2_refine", "status": "ok", "correct": true, "speedup": 2.2455, "speedup_lo95": 2.233, "ref_ms": 5.33614, "cand_ms": 2.37634, "score_pass": 2.2455, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2742, "tokens_out": 2867, "ctx_chars": 4920, "ctx_sha": "0b7bb1e527508bca", "wall_s": 232.6}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0039/P2_consult", "status": "ok", "correct": true, "speedup": 1.2457, "speedup_lo95": 1.2447, "ref_ms": 5.12516, "cand_ms": 4.11413, "score_pass": 1.2457, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12999, "tokens_out": 4139, "ctx_chars": 16089, "ctx_sha": "0a627ffc5a6a98ab", "wall_s": 407.2}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0039/P2_audit", "status": "ok", "correct": true, "speedup": 1.851, "speedup_lo95": 1.5185, "ref_ms": 5.17315, "cand_ms": 2.79472, "score_pass": 1.851, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12942, "tokens_out": 3697, "ctx_chars": 16401, "ctx_sha": "afd8a514ed425663", "wall_s": 335.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0039/P2_prior", "status": "ok", "correct": true, "speedup": 1.7318, "speedup_lo95": 1.7288, "ref_ms": 5.15418, "cand_ms": 2.97625, "score_pass": 1.7318, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2858, "tokens_out": 2862, "ctx_chars": 5548, "ctx_sha": "773cee159d7655cc", "wall_s": 281.0}, "S": {"role": "S", "run_dir": "ra-w1-0039/S", "status": "ok", "correct": true, "speedup": 1.3526, "speedup_lo95": 1.2956, "ref_ms": 5.22107, "cand_ms": 3.86, "score_pass": 1.3526, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9570, "tokens_out": 2956, "ctx_chars": 10864, "ctx_sha": "0b6222244e19e377", "wall_s": 321.3}}, "codes": {"P1": "66043f12083d46afadd02338", "P2_refine": "56a3e5139aab0c91c7b2ac96", "P2_consult": "b24b7689452fe5e38f267682", "P2_audit": "c68778aa48e7479286fc034a", "P2_prior": "664a62f7c6e14e5173ad375d", "S": "560dbc39a65f32dda0d2d26a"}, "infra_any": false, "t_start": 1791536123.7064233, "t_end": 1791536910.8753994}
{"block": "ra-w1-0034", "task": "L3_025", "reader": "gpt-6-astra", "rep": 1, "family": "network", "p1_failed": false, "arms": {"bare": {"role": "P1", "score": 1.4446, "final": {"score": 1.4446, "status": "ok", "correct": true, "speedup": 1.4446, "speedup_lo95": 1.439, "rid": "rk-final|ra-w1-0034|cdb46bc358f9755e680c4e6b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4574, "code_sha": "cdb46bc358f9755e680c4e6b"}, "show_all": {"role": "S", "score": 1.3856, "final": {"score": 1.3856, "status": "ok", "correct": true, "speedup": 1.3856, "speedup_lo95": 1.3745, "rid": "rk-final|ra-w1-0034|f1afc9ae4f2319c65f18e46d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.1686, "code_sha": "f1afc9ae4f2319c65f18e46d"}, "bare2": {"role": "P2_refine", "score": 1.6999, "final": {"score": 1.6999, "status": "ok", "correct": true, "speedup": 1.6999, "speedup_lo95": 1.5022, "rid": "rk-final|ra-w1-0034|4b94a6c7495851318c6a79c8", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6957, "code_sha": "4b94a6c7495851318c6a79c8"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.9617, "final": {"score": 1.9617, "status": "ok", "correct": true, "speedup": 1.9617, "speedup_lo95": 1.7501, "rid": "rk-final|ra-w1-0034|31873d966acf404f3414ac93", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9772, "code_sha": "31873d966acf404f3414ac93"}, "audit_then_consult": {"role": "P1", "score": 1.4446, "final": {"score": 1.4446, "status": "ok", "correct": true, "speedup": 1.4446, "speedup_lo95": 1.439, "rid": "rk-final|ra-w1-0034|cdb46bc358f9755e680c4e6b", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.4574, "code_sha": "cdb46bc358f9755e680c4e6b"}, "prior_then_consult": {"role": "P2_prior", "score": 1.842, "final": {"score": 1.842, "status": "ok", "correct": true, "speedup": 1.842, "speedup_lo95": 1.7546, "rid": "rk-final|ra-w1-0034|1e109877e3917b86c2451a5f", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.8875, "code_sha": "1e109877e3917b86c2451a5f"}, "consult_on_failure": {"role": "P2_refine", "score": 1.6999, "final": {"score": 1.6999, "status": "ok", "correct": true, "speedup": 1.6999, "speedup_lo95": 1.5022, "rid": "rk-final|ra-w1-0034|4b94a6c7495851318c6a79c8", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.6957, "code_sha": "4b94a6c7495851318c6a79c8"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0034/P1", "status": "ok", "correct": true, "speedup": 1.4574, "speedup_lo95": 1.41, "ref_ms": 20.80901, "cand_ms": 14.27792, "score_pass": 1.4574, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1575, "tokens_out": 4032, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 346.1}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0034/P2_refine", "status": "ok", "correct": true, "speedup": 1.6957, "speedup_lo95": 1.4511, "ref_ms": 20.77085, "cand_ms": 12.24904, "score_pass": 1.6957, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 4290, "tokens_out": 4180, "ctx_chars": 8992, "ctx_sha": "3c1a4ba2d5b4693b", "wall_s": 197.8}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0034/P2_consult", "status": "ok", "correct": true, "speedup": 1.9772, "speedup_lo95": 1.9716, "ref_ms": 20.77214, "cand_ms": 10.50567, "score_pass": 1.9772, "error": "", "infra": false, "infra_reruns": 0, "calls": 3, "tokens_in": 28108, "tokens_out": 11082, "ctx_chars": 15810, "ctx_sha": "389b2586290dd086", "wall_s": 544.5}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0034/P2_audit", "status": "ok", "correct": true, "speedup": 1.4228, "speedup_lo95": 1.4187, "ref_ms": 20.76631, "cand_ms": 14.59542, "score_pass": 1.4228, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 15390, "tokens_out": 7022, "ctx_chars": 16122, "ctx_sha": "eebcf6a6eca9f636", "wall_s": 414.6}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0034/P2_prior", "status": "ok", "correct": true, "speedup": 1.8875, "speedup_lo95": 1.8866, "ref_ms": 20.70588, "cand_ms": 10.97002, "score_pass": 1.8875, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 4406, "tokens_out": 4426, "ctx_chars": 9620, "ctx_sha": "e01f43f77688cc54", "wall_s": 445.2}, "S": {"role": "S", "run_dir": "ra-w1-0034/S", "status": "ok", "correct": true, "speedup": 1.1686, "speedup_lo95": 1.1662, "ref_ms": 20.79084, "cand_ms": 17.79195, "score_pass": 1.1686, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 8657, "tokens_out": 4673, "ctx_chars": 6513, "ctx_sha": "1aa698d3a9589f25", "wall_s": 293.6}}, "codes": {"P1": "cdb46bc358f9755e680c4e6b", "P2_refine": "4b94a6c7495851318c6a79c8", "P2_consult": "31873d966acf404f3414ac93", "P2_audit": "1b21c3bee439b1f0b6bb350a", "P2_prior": "1e109877e3917b86c2451a5f", "S": "f1afc9ae4f2319c65f18e46d"}, "infra_any": false, "t_start": 1791535844.6919358, "t_end": 1791536993.009161}
{"block": "ra-w1-0040", "task": "L1_081", "reader": "gpt-6-astra", "rep": 0, "family": "conv_transpose", "p1_failed": true, "arms": {"bare": {"role": "P1", "score": 0.4992, "final": {"score": 0.4992, "status": "ok", "correct": true, "speedup": 0.4992, "speedup_lo95": 0.4982, "rid": "rk-final|ra-w1-0040|cc0a99f239d2365f71712d48", "infra": false, "tries": 1, "error": ""}, "score_pass": 0.4995, "code_sha": "cc0a99f239d2365f71712d48"}, "show_all": {"role": "S", "score": 1.0779, "final": {"score": 1.0779, "status": "ok", "correct": true, "speedup": 1.0779, "speedup_lo95": 1.0752, "rid": "rk-final|ra-w1-0040|2e937db9a63b7182af59fc6c", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.091, "code_sha": "2e937db9a63b7182af59fc6c"}, "bare2": {"role": "P2_refine", "score": 2.1386, "final": {"score": 2.1386, "status": "ok", "correct": true, "speedup": 2.1386, "speedup_lo95": 2.1367, "rid": "rk-final|ra-w1-0040|d9faf52fe3f69c8209fb531b", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.0758, "code_sha": "d9faf52fe3f69c8209fb531b"}, "attempt_then_consult": {"role": "P2_consult", "score": 1.096, "final": {"score": 1.096, "status": "ok", "correct": true, "speedup": 1.096, "speedup_lo95": 1.0943, "rid": "rk-final|ra-w1-0040|4c2c60e39266ddc7a607703d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.104, "code_sha": "4c2c60e39266ddc7a607703d"}, "audit_then_consult": {"role": "P2_audit", "score": 2.1281, "final": {"score": 2.1281, "status": "ok", "correct": true, "speedup": 2.1281, "speedup_lo95": 2.1245, "rid": "rk-final|ra-w1-0040|f79bb9ed53d477ecf60b5efa", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.9081, "code_sha": "f79bb9ed53d477ecf60b5efa"}, "prior_then_consult": {"role": "P2_prior", "score": 1.9629, "final": {"score": 1.9629, "status": "ok", "correct": true, "speedup": 1.9629, "speedup_lo95": 1.9587, "rid": "rk-final|ra-w1-0040|007493d8dbe37817e0c353ed", "infra": false, "tries": 1, "error": ""}, "score_pass": 2.1384, "code_sha": "007493d8dbe37817e0c353ed"}, "consult_on_failure": {"role": "P2_consult", "score": 1.096, "final": {"score": 1.096, "status": "ok", "correct": true, "speedup": 1.096, "speedup_lo95": 1.0943, "rid": "rk-final|ra-w1-0040|4c2c60e39266ddc7a607703d", "infra": false, "tries": 1, "error": ""}, "score_pass": 1.104, "code_sha": "4c2c60e39266ddc7a607703d"}}, "episodes": {"P1": {"role": "P1", "run_dir": "ra-w1-0040/P1", "status": "ok", "correct": true, "speedup": 0.4995, "speedup_lo95": 0.4992, "ref_ms": 2.44828, "cand_ms": 4.90186, "score_pass": 0.4995, "error": "ok", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 1282, "tokens_out": 2710, "ctx_chars": 0, "ctx_sha": "e3b0c44298fc1c14", "wall_s": 163.2}, "P2_refine": {"role": "P2_refine", "run_dir": "ra-w1-0040/P2_refine", "status": "ok", "correct": true, "speedup": 2.0758, "speedup_lo95": 2.066, "ref_ms": 2.59382, "cand_ms": 1.24957, "score_pass": 2.0758, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2747, "tokens_out": 2484, "ctx_chars": 5120, "ctx_sha": "1546fc3a4b12dd64", "wall_s": 188.5}, "P2_consult": {"role": "P2_consult", "run_dir": "ra-w1-0040/P2_consult", "status": "ok", "correct": true, "speedup": 1.104, "speedup_lo95": 1.0983, "ref_ms": 2.48108, "cand_ms": 2.24744, "score_pass": 1.104, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 12646, "tokens_out": 2704, "ctx_chars": 16600, "ctx_sha": "f2dfd9839487fe21", "wall_s": 230.3}, "P2_audit": {"role": "P2_audit", "run_dir": "ra-w1-0040/P2_audit", "status": "ok", "correct": true, "speedup": 1.9081, "speedup_lo95": 1.894, "ref_ms": 2.59748, "cand_ms": 1.36127, "score_pass": 1.9081, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 13238, "tokens_out": 3331, "ctx_chars": 16912, "ctx_sha": "fb440285075f79b6", "wall_s": 268.0}, "P2_prior": {"role": "P2_prior", "run_dir": "ra-w1-0040/P2_prior", "status": "ok", "correct": true, "speedup": 2.1384, "speedup_lo95": 2.0574, "ref_ms": 2.69862, "cand_ms": 1.26198, "score_pass": 2.1384, "error": "", "infra": false, "infra_reruns": 0, "calls": 1, "tokens_in": 2863, "tokens_out": 2316, "ctx_chars": 5748, "ctx_sha": "383ac4203a35bccf", "wall_s": 210.6}, "S": {"role": "S", "run_dir": "ra-w1-0040/S", "status": "ok", "correct": true, "speedup": 1.091, "speedup_lo95": 1.0891, "ref_ms": 2.45999, "cand_ms": 2.25485, "score_pass": 1.091, "error": "", "infra": false, "infra_reruns": 0, "calls": 2, "tokens_in": 9729, "tokens_out": 2685, "ctx_chars": 11175, "ctx_sha": "91e4a03c424736b2", "wall_s": 387.0}}, "codes": {"P1": "cc0a99f239d2365f71712d48", "P2_refine": "d9faf52fe3f69c8209fb531b", "P2_consult": "4c2c60e39266ddc7a607703d", "P2_audit": "f79bb9ed53d477ecf60b5efa", "P2_prior": "007493d8dbe37817e0c353ed", "S": "2e937db9a63b7182af59fc6c"}, "infra_any": false, "t_start": 1791536449.6010227, "t_end": 1791537017.2434108}