{ "benchmark": "cb-mixture-36", "upstream_commit": "63b9d44d9f39a02fccf5bf0052db48a917a011fd", "pair_pass_definition": "Both features pass in the same official eval.json.", "duration_field": "result.json:duration_seconds", "duration_scope": "Pair harness execution including environment setup, workers, merge, optional repair and cleanup; excludes Slurm/pair-queue waiting, campaign image staging and official evaluation.", "p99_estimator": "Linear interpolation, index 0.99*(n-1); equivalent to statistics.quantiles(n=100, method=\"inclusive\")[98].", "aggregation": "Compute pair pass rate, median (P50), arithmetic mean duration and linearly interpolated P99 over all 36 pairs in each trial, then take the arithmetic mean of each metric across the three trials. No success-only filtering.", "campaigns": [ { "campaign": "20260923-cb36x3-676daba", "trials": [ { "trial": 1, "run_name": "20260923T090735Z-cooperbench-round1-676dabac", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 350.0669367313385, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 442.7021653652191, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 317.94587898254395, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 382.68527460098267, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 517.0933554172516, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 1169.5975320339203, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 803.7442815303802, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3728.1857380867004, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 3652.4873321056366, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 1146.8177471160889, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 1584.7955467700958, "both_passed": false, "feature_passes": 1 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1293.920891046524, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 613.6479938030243, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 384.2351129055023, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 428.4817085266113, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1587.016576051712, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 796.2037806510925, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 869.7595458030701, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 400.4736998081207, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 701.4923243522644, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 828.1815431118011, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 3053.1077761650085, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3683.1951336860657, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 2140.4818699359894, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 1344.1589438915253, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3614.3644795417786, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3669.122537136078, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3624.3900334835052, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3621.1405868530273, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 1071.7680504322052, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 3908.8936715126038, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 1819.355842590332, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1003.3937249183655, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 1715.0642976760864, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1029.6488616466522, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3611.8151438236237, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 3, "pair_pass_rate_percent": 8.333333333333332, "p99_seconds": 3845.645894813538, "p50_seconds": 1158.2076395750046, "mean_duration_seconds": 1691.9287755025757 }, { "trial": 2, "run_name": "20260923T092313Z-cooperbench-round2-676dabac", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 631.5905063152313, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 479.2437791824341, "both_passed": false, "feature_passes": 1 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 387.4408013820648, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 1005.009578704834, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 1406.8576414585114, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3667.1713678836823, "both_passed": true, "feature_passes": 2 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 2165.379348039627, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3704.865796804428, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 728.8760607242584, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 1275.13250374794, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3614.863416671753, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 3634.1852695941925, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 938.8929917812347, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 420.6827805042267, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 1371.567523241043, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1357.246146440506, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 966.5811433792114, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 1243.4982545375824, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 3134.9779613018036, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 1103.6093308925629, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 816.4639139175415, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 1713.6083738803864, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3620.6880362033844, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1649.7473661899567, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 1941.9705843925476, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 2487.3949308395386, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3628.526165008545, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3618.6166009902954, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3017.2345159053802, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 1017.8809232711792, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 2912.875007867813, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 629.0038433074951, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 2798.785414457321, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 3614.900341272354, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 584.5594358444214, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 1368.9562180042267, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 3, "pair_pass_rate_percent": 8.333333333333332, "p99_seconds": 3691.672746682167, "p50_seconds": 1389.2125823497772, "mean_duration_seconds": 1907.191218720542 }, { "trial": 3, "run_name": "20260923T092820Z-cooperbench-round3-676dabac", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 550.5605359077454, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 690.5038509368896, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 566.4519176483154, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 744.3028192520142, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 804.1732053756714, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 1960.6269381046295, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 3010.0840780735016, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 1914.5441012382507, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 925.3251042366028, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 960.1588213443756, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3615.798973798752, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1344.1949999332428, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 562.5097749233246, "both_passed": true, "feature_passes": 2 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 311.9901473522186, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 1619.4445230960846, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 2207.6869642734528, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 1957.1050810813904, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 3614.091470718384, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 1740.482876777649, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 1070.421552658081, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 747.7795135974884, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 2218.175089597702, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3100.071442604065, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1550.3488354682922, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 1442.3427448272705, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3610.743460416794, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 2046.078696489334, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3615.189081430435, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 2937.327637195587, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 1340.308688879013, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 1381.7042105197906, "both_passed": false, "feature_passes": 1 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 1049.4431192874908, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1065.5227706432343, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 901.8333597183228, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1267.3745551109314, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3611.225150346756, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 3, "pair_pass_rate_percent": 8.333333333333332, "p99_seconds": 3615.585511469841, "p50_seconds": 1412.0234776735306, "mean_duration_seconds": 1723.7757248017524 } ], "setting": "Baseline", "source_commit": "676daba", "mean_pair_pass_rate_percent": 8.333333333333332, "mean_trial_p99_seconds": 3717.634717655182, "mean_trial_p50_seconds": 1319.8145665327709, "mean_duration_seconds": 1774.29857300829 }, { "campaign": "20260923-cb36x3-worker-thinking-cd42039", "trials": [ { "trial": 1, "run_name": "20260923T200444Z-cooperbench-round1-cd42039c", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 428.54535388946533, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 517.9565114974976, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 432.7042717933655, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 455.56829142570496, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 668.3506398200989, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 1587.2560250759125, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 2067.669842004776, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3667.551658153534, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 1074.5515458583832, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 797.0157225131989, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3615.0472412109375, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 3011.7962834835052, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 763.7086136341095, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 560.9774878025055, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 1173.1562676429749, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1771.4913938045502, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 602.3942275047302, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 963.634170293808, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 1109.7649052143097, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 914.4823279380798, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 756.087057352066, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 3639.645693540573, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 1469.993658065796, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1722.8338181972504, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 411.69291639328003, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3739.5686433315277, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 1715.1273999214172, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3627.1268730163574, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3616.087582588196, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 880.5421929359436, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 3659.008068084717, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 1002.5207960605621, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1480.54740858078, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 3287.268330335617, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 3252.351133584976, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3198.7329199314117, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 2, "pair_pass_rate_percent": 5.555555555555555, "p99_seconds": 3714.36269851923, "p50_seconds": 1321.5749628543854, "mean_duration_seconds": 1767.8543686800533 }, { "trial": 2, "run_name": "20260923T200939Z-cooperbench-round2-cd42039c", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 726.6839997768402, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 527.6887123584747, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 774.4790806770325, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 690.0901489257812, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 820.320309638977, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3667.480740070343, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 1506.432213306427, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3680.5261673927307, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 988.3914816379547, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 845.8099868297577, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3613.0959787368774, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1751.4517407417297, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 597.1549999713898, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 594.163875579834, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 794.9345302581787, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1465.0107777118683, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 1431.7857735157013, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 1097.613410949707, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 598.5269362926483, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 757.7043418884277, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 1043.0188248157501, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 1354.980288028717, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 1506.665060043335, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1256.1558723449707, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 1221.1350808143616, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3666.196441411972, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3611.35324549675, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3640.5967383384705, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3622.302421092987, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 1493.66792345047, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 3622.3246834278107, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 3623.4648592472076, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 2637.7263782024384, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 3486.2655477523804, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 2375.408439874649, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3625.6589069366455, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 2, "pair_pass_rate_percent": 5.555555555555555, "p99_seconds": 3675.960267829895, "p50_seconds": 1448.3982756137848, "mean_duration_seconds": 1908.7851643760998 }, { "trial": 3, "run_name": "20260923T201125Z-cooperbench-round3-cd42039c", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 3585.0938317775726, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 714.7707409858704, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 859.1747753620148, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 679.9201016426086, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 1130.1241142749786, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3672.778954267502, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 3734.696882724762, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3704.6128482818604, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 3366.535251379013, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 907.5013682842255, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3634.096300125122, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 3646.338976621628, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 1070.1407146453857, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 694.2316331863403, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 541.1101586818695, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1699.431562423706, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 755.2419509887695, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 587.3935511112213, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 794.6577076911926, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 1234.9374978542328, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 2499.913338661194, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 3617.4828839302063, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 1258.074914932251, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 3619.512458086014, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 2435.1267302036285, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3610.5516300201416, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3620.232703447342, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3634.660802602768, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3614.818916320801, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 3228.4958238601685, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 2850.9167773723602, "both_passed": true, "feature_passes": 2 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 1542.4623997211456, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 587.0655717849731, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 2366.956952571869, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1383.9971237182617, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 862.0857765674591, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 4, "pair_pass_rate_percent": 11.11111111111111, "p99_seconds": 3724.1674706697463, "p50_seconds": 2033.1942574977875, "mean_duration_seconds": 2159.5873257252906 } ], "setting": "Worker thinking", "source_commit": "cd42039", "mean_pair_pass_rate_percent": 7.407407407407407, "mean_trial_p99_seconds": 3704.8301456729573, "mean_trial_p50_seconds": 1601.0558319886525, "mean_duration_seconds": 1945.408952927148 }, { "campaign": "20260923-cb36x3-no-coordinator-0aad5fe", "trials": [ { "trial": 1, "run_name": "20260924T002338Z-cooperbench-round1-0aad5fe0", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 734.5935370922089, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 473.7316925525665, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 465.2739293575287, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 506.7366695404053, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 503.6271986961365, "both_passed": true, "feature_passes": 2 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3659.8552074432373, "both_passed": false, "feature_passes": 1 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 1959.3992972373962, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3726.026087284088, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 460.7417573928833, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 552.755366563797, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 2623.9930186271667, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 3633.962757587433, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 658.5611379146576, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 418.5236818790436, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 427.898193359375, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 3627.6365032196045, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 3638.4255373477936, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 2921.7378220558167, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 1360.9884631633759, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 3089.1628997325897, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 2607.2633213996887, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 1846.9155659675598, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 1737.598346710205, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1134.535796880722, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 3639.3465576171875, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3615.734249353409, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3849.399467229843, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3644.4201486110687, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3661.9113416671753, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 3613.219897031784, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 2698.6161851882935, "both_passed": false, "feature_passes": 1 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 3633.473172903061, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 2219.098274230957, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 1307.349463224411, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 3627.5061242580414, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3612.501363515854, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 4, "pair_pass_rate_percent": 11.11111111111111, "p99_seconds": 3806.218784248829, "p50_seconds": 2615.6281700134277, "mean_duration_seconds": 2274.7922231621214 }, { "trial": 2, "run_name": "20260924T003709Z-cooperbench-round2-0aad5fe0", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 1344.4642980098724, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 1579.4531686306, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 569.4834928512573, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 1089.241467475891, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 3809.8092975616455, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3710.214107275009, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 3727.6566421985626, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3709.7337386608124, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 3673.399406194687, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 3630.006444454193, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3627.067757129669, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 2877.1928985118866, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 1678.0847823619843, "both_passed": false, "feature_passes": 0 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 1676.7383840084076, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 3440.350634098053, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 3313.4843299388885, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 3633.950443983078, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 3534.4420931339264, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 2543.946485519409, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 2425.3133692741394, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 786.076621055603, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 1729.9852423667908, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3634.5270726680756, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1150.1689069271088, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 3695.0442831516266, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 1944.2595419883728, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3687.8716957569122, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3615.656655550003, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3618.711332321167, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 2808.076009273529, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 2975.629183769226, "both_passed": false, "feature_passes": 1 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 3440.3749849796295, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 2587.769184112549, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 1911.7850198745728, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 2192.1173770427704, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3167.9037363529205, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 3, "pair_pass_rate_percent": 8.333333333333332, "p99_seconds": 3781.0558681845664, "p50_seconds": 3071.7664600610733, "mean_duration_seconds": 2737.2219469017455 }, { "trial": 3, "run_name": "20260924T003724Z-cooperbench-round3-0aad5fe0", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 2864.5203964710236, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 2366.182831287384, "both_passed": false, "feature_passes": 1 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 459.61885237693787, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 1670.5950543880463, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 3999.773075580597, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3036.5606756210327, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 3707.7681093215942, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3741.2369270324707, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 3344.3122656345367, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 2966.714864015579, "both_passed": true, "feature_passes": 2 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3626.923630475998, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 3637.5244936943054, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 1690.8042719364166, "both_passed": true, "feature_passes": 2 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 999.6900444030762, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 3321.569233894348, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 1302.6982727050781, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 1399.5517790317535, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 2241.3331813812256, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 2644.8594648838043, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 2561.3599474430084, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 1802.8057510852814, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 3651.3745844364166, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3488.3292050361633, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 987.1708204746246, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 3632.200197696686, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 2992.323297739029, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3640.4353177547455, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3663.2644028663635, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3636.7662301063538, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 3625.0732333660126, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 2366.5671861171722, "both_passed": true, "feature_passes": 2 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 3621.7936527729034, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1837.5663239955902, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 1931.2523713111877, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 2125.882107257843, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3631.6637194156647, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 5, "pair_pass_rate_percent": 13.88888888888889, "p99_seconds": 3909.2854235887526, "p50_seconds": 2979.519080877304, "mean_duration_seconds": 2728.2796048058403 } ], "setting": "No coordinator", "source_commit": "0aad5fe", "mean_pair_pass_rate_percent": 11.11111111111111, "mean_trial_p99_seconds": 3832.1866920073826, "mean_trial_p50_seconds": 2888.971236983935, "mean_duration_seconds": 2580.097924956569 }, { "campaign": "20260925-cb36x3-repair-ef8b4d8", "trials": [ { "trial": 1, "run_name": "20260925T074457Z-cooperbench-round1-ef8b4d84", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 681.7176780700684, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 529.8660991191864, "both_passed": false, "feature_passes": 1 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 568.0166118144989, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 678.9638948440552, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 620.0596141815186, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 3809.4911081790924, "both_passed": false, "feature_passes": 1 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 950.5473585128784, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3837.5125918388367, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 578.0849933624268, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 1082.0175621509552, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3688.2526450157166, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1865.7871706485748, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 704.1552312374115, "both_passed": true, "feature_passes": 2 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 426.38062596321106, "both_passed": false, "feature_passes": 1 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 550.5901520252228, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 749.0829491615295, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 2201.338487625122, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 521.1523637771606, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 537.2545404434204, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 1430.6186485290527, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 1507.6434762477875, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 1038.0236687660217, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 2142.3167865276337, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1313.6675021648407, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 558.6679358482361, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3630.866651058197, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3610.2668550014496, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3611.2705767154694, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 1778.8653514385223, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 785.1881709098816, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 1477.9603281021118, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 924.281626701355, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1172.0670628547668, "both_passed": false, "feature_passes": 1 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 2080.3966670036316, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1797.8607876300812, "both_passed": false, "feature_passes": 1 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3697.88605761528, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 10, "pair_pass_rate_percent": 27.77777777777778, "p99_seconds": 3827.705072557926, "p50_seconds": 1127.042312502861, "mean_duration_seconds": 1587.1699953079224 }, { "trial": 2, "run_name": "20260925T085108Z-cooperbench-round2-ef8b4d84", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 704.3153374195099, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 637.2737197875977, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 425.50498056411743, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 617.4610297679901, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 729.2116186618805, "both_passed": false, "feature_passes": 1 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 1479.3617491722107, "both_passed": false, "feature_passes": 1 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 1350.2288098335266, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 3889.558744907379, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 2478.2101306915283, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 646.8880224227905, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 3696.1122035980225, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1381.6854872703552, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 952.1217103004456, "both_passed": true, "feature_passes": 2 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 791.4710500240326, "both_passed": false, "feature_passes": 1 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 983.5656571388245, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 629.8372716903687, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 1593.16690659523, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 1015.0513126850128, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 1217.1532878875732, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 619.0089588165283, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 669.2616271972656, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 2079.2492246627808, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 1354.3218586444855, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 1731.6752269268036, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 605.2678451538086, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 3643.7138636112213, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 3687.1500985622406, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3623.8203682899475, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3617.8748455047607, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 2429.642636537552, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 929.9796671867371, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 1193.2930340766907, "both_passed": false, "feature_passes": 0 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1339.5556640625, "both_passed": true, "feature_passes": 2 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 1965.8447127342224, "both_passed": false, "feature_passes": 1 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1301.682373046875, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 3325.1625204086304, "both_passed": false, "feature_passes": 1 } ], "pair_passes": 11, "pair_pass_rate_percent": 30.555555555555557, "p99_seconds": 3821.852455449104, "p50_seconds": 1320.6190185546875, "mean_duration_seconds": 1648.185654328929 }, { "trial": 3, "run_name": "20260925T085233Z-cooperbench-round3-ef8b4d84", "pairs": [ { "pair": "dottxt_ai_outlines_task:1655:6,7", "duration_seconds": 818.5993168354034, "both_passed": false, "feature_passes": 0 }, { "pair": "dottxt_ai_outlines_task:1655:7,10", "duration_seconds": 820.0913324356079, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:4,6", "duration_seconds": 635.4622476100922, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,6", "duration_seconds": 489.3585522174835, "both_passed": true, "feature_passes": 2 }, { "pair": "dottxt_ai_outlines_task:1706:5,8", "duration_seconds": 782.8902654647827, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,4", "duration_seconds": 2127.3031203746796, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8394:3,5", "duration_seconds": 2656.200110435486, "both_passed": false, "feature_passes": 1 }, { "pair": "dspy_task:8394:4,5", "duration_seconds": 2995.0243105888367, "both_passed": false, "feature_passes": 0 }, { "pair": "dspy_task:8635:4,6", "duration_seconds": 823.510755777359, "both_passed": false, "feature_passes": 0 }, { "pair": "go_chi_task:26:1,2", "duration_seconds": 1923.6648292541504, "both_passed": false, "feature_passes": 1 }, { "pair": "go_chi_task:56:1,5", "duration_seconds": 1257.4263756275177, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:3997:2,4", "duration_seconds": 1350.0358390808105, "both_passed": false, "feature_passes": 0 }, { "pair": "huggingface_datasets_task:6252:4,6", "duration_seconds": 606.2247025966644, "both_passed": true, "feature_passes": 2 }, { "pair": "llama_index_task:18813:1,5", "duration_seconds": 564.9496309757233, "both_passed": false, "feature_passes": 1 }, { "pair": "openai_tiktoken_task:0:1,5", "duration_seconds": 382.2788875102997, "both_passed": false, "feature_passes": 0 }, { "pair": "openai_tiktoken_task:0:3,6", "duration_seconds": 722.0185921192169, "both_passed": true, "feature_passes": 2 }, { "pair": "openai_tiktoken_task:0:6,8", "duration_seconds": 580.0684368610382, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_click_task:2068:1,6", "duration_seconds": 1355.8029096126556, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:2,10", "duration_seconds": 451.66096115112305, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:3,12", "duration_seconds": 770.5387759208679, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_click_task:2068:7,9", "duration_seconds": 560.6647856235504, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2068:9,11", "duration_seconds": 856.3614439964294, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_click_task:2800:1,3", "duration_seconds": 3679.973521232605, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:1,7", "duration_seconds": 851.4713928699493, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1465:2,3", "duration_seconds": 1598.7104251384735, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1559:4,6", "duration_seconds": 1459.9323811531067, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1559:4,7", "duration_seconds": 1745.8403124809265, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:5,8", "duration_seconds": 3690.4183809757233, "both_passed": false, "feature_passes": 1 }, { "pair": "pallets_jinja_task:1559:7,9", "duration_seconds": 3610.327739238739, "both_passed": false, "feature_passes": 0 }, { "pair": "pallets_jinja_task:1621:2,9", "duration_seconds": 619.7805478572845, "both_passed": true, "feature_passes": 2 }, { "pair": "pallets_jinja_task:1621:4,6", "duration_seconds": 3739.75172996521, "both_passed": false, "feature_passes": 1 }, { "pair": "pillow_task:290:2,3", "duration_seconds": 911.5621247291565, "both_passed": false, "feature_passes": 1 }, { "pair": "pillow_task:290:4,5", "duration_seconds": 1097.115418434143, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:2,3", "duration_seconds": 2252.2089223861694, "both_passed": false, "feature_passes": 0 }, { "pair": "samuelcolvin_dirty_equals_task:43:3,7", "duration_seconds": 1674.2688963413239, "both_passed": false, "feature_passes": 1 }, { "pair": "samuelcolvin_dirty_equals_task:43:7,9", "duration_seconds": 1776.4342868328094, "both_passed": false, "feature_passes": 0 } ], "pair_passes": 8, "pair_pass_rate_percent": 22.22222222222222, "p99_seconds": 3722.48505781889, "p50_seconds": 1004.3387715816498, "mean_duration_seconds": 1451.053673936261 } ], "setting": "Merge repair", "source_commit": "ef8b4d8", "mean_pair_pass_rate_percent": 26.85185185185185, "mean_trial_p99_seconds": 3790.6808619419735, "mean_trial_p50_seconds": 1150.6667008797328, "mean_duration_seconds": 1562.1364411910374 } ], "p50_estimator": "Median of 36 durations: arithmetic mean of sorted observations 18 and 19 (one-based)." }