{ "scope": "Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)", "unit": "pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt.", "generated_at": "2026-09-03T22:24:37.984682+00:00", "n_tasks_in_set": 113, "latest_job": { "name": "20260901-deep-swe-1-1-gpt-6-astra", "finished_at": "2026-09-01T07:35:13Z" }, "rows": [ { "model": "gpt-6-astra", "harness": "mini-swe-agent", "provider": "openai", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_6_astra_xhigh", "source": "deep-swe", "cost_basis": "Expected launch pricing at all context lengths: $12/M uncached input, $15/M cache writes, $1.20/M cache reads, $50/M output, $2/M compute units.", "pass_rate": 0.7411504424778761, "pass_at_1": 0.7411504424778761, "pass_at_4": 0.8053097345132744, "n_passed": 335, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "completed_by_attempt": [ 113, 113, 113, 113 ], "pass_rate_by_attempt": [ 0.7079646017699115, 0.7345132743362832, 0.7787610619469026, 0.7433628318584071 ], "ci_passed": 335, "ci_attempted": 452, "ci_lo": 0.7124964807371247, "ci_hi": 0.7698044042186275, "ci_half": 0.02865396174075141, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 6.52377356460177, "median_cost_usd": 5.6717151, "mean_output_tokens": 29557.327433628318, "median_output_tokens": 28542.5, "mean_input_tokens": 1456927.0929203539, "median_input_tokens": 1163918.5, "mean_uncached_input_tokens": 295.3871681415929, "median_uncached_input_tokens": 87.0, "mean_cache_tokens": 1326921.6836283186, "mean_cache_read_tokens": 1326921.6836283186, "median_cache_read_tokens": 1035975.0, "mean_cache_write_tokens": 129710.02212389381, "median_cache_write_tokens": 103995.0, "mean_reasoning_tokens": 11255.66592920354, "median_reasoning_tokens": 10434.0, "mean_compute_units": 752203.0973451327, "median_compute_units": 662371.5, "mean_duration_seconds": 1132.4004424778761, "median_duration_seconds": 959.0, "mean_agent_steps": 28.754424778761063, "median_agent_steps": 26.0, "median_peak_context_tokens": null, "median_output_tokens_to_pass": 28361 }, { "model": "gemini-3-8-flash", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gemini_3_8_flash_high", "source": "deep-swe", "pass_rate": 0.738255033557047, "pass_at_1": 0.738255033557047, "pass_at_4": 0.8584070796460177, "n_passed": 330, "n_attempted": 447, "n_tasks_attempted": 113, "n_tasks_passed_any": 97, "ci_passed": 330, "ci_attempted": 447, "ci_lo": 0.7240819014906883, "ci_hi": 0.7524281656234058, "ci_half": 0.014173132066358783, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.362349413758389, "median_cost_usd": 2.1098912999999997, "mean_output_tokens": 143242.6644295302, "median_output_tokens": 138377.0, "mean_input_tokens": 21734449.25279642, "median_input_tokens": 18026661.0, "mean_cache_tokens": 21456039.230425056, "mean_duration_seconds": 686.7691519843399, "median_duration_seconds": 604.643838, "mean_agent_steps": 166.3131991051454, "median_agent_steps": 161.0, "median_peak_context_tokens": 215558.0, "median_output_tokens_to_pass": 136617.5 }, { "model": "claude-opus-5", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_claude_opus_5_max", "source": "deep-swe", "pass_rate": 0.7364864864864865, "pass_at_1": 0.7364864864864865, "pass_at_4": 0.8849557522123894, "n_passed": 327, "n_attempted": 444, "n_tasks_attempted": 113, "n_tasks_passed_any": 100, "ci_passed": 327, "ci_attempted": 444, "ci_lo": 0.6977633822227692, "ci_hi": 0.7752095907502038, "ci_half": 0.03872310426371729, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 11.837583271396396, "median_cost_usd": 10.4281505, "mean_output_tokens": 117565.6936936937, "median_output_tokens": 113366.5, "mean_input_tokens": 15025834.39864865, "median_input_tokens": 12130307.5, "mean_cache_tokens": 14784785.963963963, "mean_duration_seconds": 1911.819866231982, "median_duration_seconds": 1801.1237265, "mean_agent_steps": 99.0427927927928, "median_agent_steps": 90.5, "median_peak_context_tokens": 215810.0, "median_output_tokens_to_pass": 112874.0 }, { "model": "gpt-6-astra", "harness": "mini-swe-agent", "provider": "openai", "reasoning_effort": "high", "config": "mini_swe_agent_gpt_6_astra_high", "source": "deep-swe", "cost_basis": "Expected launch pricing at all context lengths: $12/M uncached input, $15/M cache writes, $1.20/M cache reads, $50/M output, $2/M compute units.", "pass_rate": 0.7323008849557522, "pass_at_1": 0.7323008849557522, "pass_at_4": 0.8230088495575221, "n_passed": 331, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 93, "completed_by_attempt": [ 113, 113, 113, 113 ], "pass_rate_by_attempt": [ 0.7610619469026548, 0.7168141592920354, 0.7610619469026548, 0.6902654867256637 ], "ci_passed": 331, "ci_attempted": 452, "ci_lo": 0.6980659243770221, "ci_hi": 0.7665358455344823, "ci_half": 0.03423496057873005, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 5.723721956194691, "median_cost_usd": 4.9622669, "mean_output_tokens": 26505.853982300883, "median_output_tokens": 25414.0, "mean_input_tokens": 1283270.8030973452, "median_input_tokens": 995020.0, "mean_uncached_input_tokens": 450.7853982300885, "median_uncached_input_tokens": 81.0, "mean_cache_tokens": 1168811.0420353983, "mean_cache_read_tokens": 1168811.0420353983, "median_cache_read_tokens": 905705.0, "mean_cache_write_tokens": 114008.97566371682, "median_cache_write_tokens": 92618.0, "mean_reasoning_tokens": 9021.692477876106, "median_reasoning_tokens": 8273.5, "mean_compute_units": 640155.9734513274, "median_compute_units": 562938.0, "mean_duration_seconds": 1038.6858407079646, "median_duration_seconds": 893.5, "mean_agent_steps": 27.424778761061948, "median_agent_steps": 25.0, "median_peak_context_tokens": null, "median_output_tokens_to_pass": 25170 }, { "model": "gpt-6-astra", "harness": "mini-swe-agent", "provider": "openai", "reasoning_effort": "max", "config": "mini_swe_agent_gpt_6_astra_max", "source": "deep-swe", "cost_basis": "Expected launch pricing at all context lengths: $12/M uncached input, $15/M cache writes, $1.20/M cache reads, $50/M output, $2/M compute units.", "pass_rate": 0.7323008849557522, "pass_at_1": 0.7323008849557522, "pass_at_4": 0.7964601769911505, "n_passed": 331, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 90, "completed_by_attempt": [ 113, 113, 113, 113 ], "pass_rate_by_attempt": [ 0.7345132743362832, 0.7433628318584071, 0.7256637168141593, 0.7256637168141593 ], "ci_passed": 331, "ci_attempted": 452, "ci_lo": 0.7239976873936956, "ci_hi": 0.7406040825178087, "ci_half": 0.008303197562056526, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 12.369032401327434, "median_cost_usd": 11.036936, "mean_output_tokens": 61148.50663716814, "median_output_tokens": 58617.5, "mean_input_tokens": 2183033.681415929, "median_input_tokens": 1767876.5, "mean_uncached_input_tokens": 294.4845132743363, "median_uncached_input_tokens": 84.0, "mean_cache_tokens": 1986676.7079646017, "mean_cache_read_tokens": 1986676.7079646017, "median_cache_read_tokens": 1598263.0, "mean_cache_write_tokens": 196062.4889380531, "median_cache_write_tokens": 165256.0, "mean_reasoning_tokens": 36555.72345132743, "median_reasoning_tokens": 35029.5, "mean_compute_units": 1991561.935840708, "median_compute_units": 1760214.0, "mean_duration_seconds": 1982.966814159292, "median_duration_seconds": 1773.5, "mean_agent_steps": 28.45353982300885, "median_agent_steps": 26.0, "median_peak_context_tokens": null, "median_output_tokens_to_pass": 58206 }, { "model": "claude-opus-5", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_claude_opus_5_xhigh", "source": "deep-swe", "pass_rate": 0.7315436241610739, "pass_at_1": 0.7315436241610739, "pass_at_4": 0.8584070796460177, "n_passed": 327, "n_attempted": 447, "n_tasks_attempted": 113, "n_tasks_passed_any": 97, "ci_passed": 327, "ci_attempted": 447, "ci_lo": 0.7009338059446892, "ci_hi": 0.7621534423774585, "ci_half": 0.030609818216384643, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 9.07220232326622, "median_cost_usd": 7.928816750000001, "mean_output_tokens": 91672.01565995526, "median_output_tokens": 87322.0, "mean_input_tokens": 11292729.463087248, "median_input_tokens": 8932820.0, "mean_cache_tokens": 11094990.232662192, "mean_duration_seconds": 1559.5597273221476, "median_duration_seconds": 1448.063854, "mean_agent_steps": 88.7248322147651, "median_agent_steps": 80.0, "median_peak_context_tokens": 173749.0, "median_output_tokens_to_pass": 87949.0 }, { "model": "claude-opus-5", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_claude_opus_5_high", "source": "deep-swe", "pass_rate": 0.7282850779510023, "pass_at_1": 0.7282850779510023, "pass_at_4": 0.8761061946902655, "n_passed": 327, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 99, "ci_passed": 327, "ci_attempted": 449, "ci_lo": 0.7088307563794007, "ci_hi": 0.7477393995226038, "ci_half": 0.019454321571601495, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 6.076084489977728, "median_cost_usd": 4.96882375, "mean_output_tokens": 64207.43429844098, "median_output_tokens": 59856.0, "mean_input_tokens": 7233879.0824053455, "median_input_tokens": 5212804.0, "mean_cache_tokens": 7085027.516703786, "mean_duration_seconds": 1163.8627881514476, "median_duration_seconds": 1006.103652, "mean_agent_steps": 72.92204899777283, "median_agent_steps": 64.0, "median_peak_context_tokens": 126517.0, "median_output_tokens_to_pass": 60897.0 }, { "model": "gpt-6-astra", "harness": "mini-swe-agent", "provider": "openai", "reasoning_effort": "medium", "config": "mini_swe_agent_gpt_6_astra_medium", "source": "deep-swe", "cost_basis": "Expected launch pricing at all context lengths: $12/M uncached input, $15/M cache writes, $1.20/M cache reads, $50/M output, $2/M compute units.", "pass_rate": 0.7278761061946902, "pass_at_1": 0.7278761061946902, "pass_at_4": 0.8230088495575221, "n_passed": 329, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 93, "completed_by_attempt": [ 113, 113, 113, 113 ], "pass_rate_by_attempt": [ 0.7345132743362832, 0.6991150442477876, 0.7168141592920354, 0.7610619469026548 ], "ci_passed": 329, "ci_attempted": 452, "ci_lo": 0.7019796153763715, "ci_hi": 0.7537725970130089, "ci_half": 0.025896490818318695, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.379527705752213, "median_cost_usd": 3.7133415999999997, "mean_output_tokens": 20361.809734513274, "median_output_tokens": 19006.0, "mean_input_tokens": 1008238.2898230088, "median_input_tokens": 779601.0, "mean_uncached_input_tokens": 293.92699115044246, "median_uncached_input_tokens": 78.0, "mean_cache_tokens": 916944.3473451327, "mean_cache_read_tokens": 916944.3473451327, "median_cache_read_tokens": 703332.5, "mean_cache_write_tokens": 91000.01548672566, "median_cache_write_tokens": 73977.5, "mean_reasoning_tokens": 5510.918141592921, "median_reasoning_tokens": 4569.0, "mean_compute_units": 446288.3230088496, "median_compute_units": 371268.0, "mean_duration_seconds": 883.9358407079646, "median_duration_seconds": 747.5, "mean_agent_steps": 26.030973451327434, "median_agent_steps": 24.0, "median_peak_context_tokens": null, "median_output_tokens_to_pass": 19006 }, { "model": "gpt-5-6-sol", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_gpt_5_6_sol_max", "source": "deep-swe", "pass_rate": 0.7266666666666667, "pass_at_1": 0.7266666666666667, "pass_at_4": 0.8584070796460177, "n_passed": 327, "n_attempted": 450, "n_tasks_attempted": 113, "n_tasks_passed_any": 97, "ci_passed": 327, "ci_attempted": 450, "ci_lo": 0.6983684441682949, "ci_hi": 0.7549648891650385, "ci_half": 0.02829822249837175, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 8.386436346666667, "median_cost_usd": 6.8414715, "mean_output_tokens": 60013.64444444444, "median_output_tokens": 58786.5, "mean_input_tokens": 7907652.344444444, "median_input_tokens": 5973865.5, "mean_cache_tokens": 7419435.235555556, "mean_duration_seconds": 1128.624910331111, "median_duration_seconds": 1013.6291865000001, "mean_agent_steps": 61.25333333333333, "median_agent_steps": 53.0, "median_peak_context_tokens": 177470.5, "median_output_tokens_to_pass": 58467.0 }, { "model": "gemini-3-8-flash", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gemini_3_8_flash_medium", "source": "deep-swe", "pass_rate": 0.7101769911504425, "pass_at_1": 0.7101769911504425, "pass_at_4": 0.831858407079646, "n_passed": 321, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 94, "ci_passed": 321, "ci_attempted": 452, "ci_lo": 0.6873689454216633, "ci_hi": 0.7329850368792218, "ci_half": 0.02280804572877925, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.9671024789823006, "median_cost_usd": 1.7342966249999998, "mean_output_tokens": 124684.36283185841, "median_output_tokens": 120488.0, "mean_input_tokens": 16863939.814159293, "median_input_tokens": 13458718.0, "mean_cache_tokens": 16519023.008849557, "mean_duration_seconds": 826.5496385464602, "median_duration_seconds": 668.1031395, "mean_agent_steps": 147.30088495575222, "median_agent_steps": 140.0, "median_peak_context_tokens": 189491.5, "median_output_tokens_to_pass": 119834.0 }, { "model": "gpt-5-6-sol", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_5_6_sol_xhigh", "source": "deep-swe", "pass_rate": 0.7073170731707317, "pass_at_1": 0.7073170731707317, "pass_at_4": 0.8584070796460177, "n_passed": 319, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 97, "ci_passed": 319, "ci_attempted": 451, "ci_lo": 0.6991259559533886, "ci_hi": 0.7155081903880748, "ci_half": 0.008191117217343103, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.703663860310422, "median_cost_usd": 4.117039, "mean_output_tokens": 40744.59201773836, "median_output_tokens": 39449.0, "mean_input_tokens": 4259318.379157428, "median_input_tokens": 3343837.0, "mean_cache_tokens": 3967289.3303769403, "mean_duration_seconds": 800.5459875388026, "median_duration_seconds": 694.028368, "mean_agent_steps": 44.0, "median_agent_steps": 39.0, "median_peak_context_tokens": 133054.0, "median_output_tokens_to_pass": 39434.0 }, { "model": "claude-fable-5", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_claude_fable_5_xhigh", "source": "deep-swe", "pass_rate": 0.6991150442477876, "pass_at_1": 0.6991150442477876, "pass_at_4": 0.8849557522123894, "n_passed": 316, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 100, "ci_passed": 316, "ci_attempted": 452, "ci_lo": 0.6666658684930744, "ci_hi": 0.7315642200025008, "ci_half": 0.03244917575471319, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 13.414521495535714, "median_cost_usd": 11.341082999999998, "mean_output_tokens": 80352.1703539823, "median_output_tokens": 76035.5, "mean_input_tokens": 7329160.601769911, "median_input_tokens": 5337563.5, "mean_cache_tokens": 7159496.181415929, "mean_duration_seconds": 1411.5830643938054, "median_duration_seconds": 1231.553523, "mean_agent_steps": 68.40044247787611, "median_agent_steps": 61.5, "median_peak_context_tokens": 141254.5, "median_output_tokens_to_pass": 75232.0 }, { "model": "claude-fable-5", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_claude_fable_5_max", "source": "deep-swe", "pass_rate": 0.6972477064220184, "pass_at_1": 0.6972477064220184, "pass_at_4": 0.8407079646017699, "n_passed": 304, "n_attempted": 436, "n_tasks_attempted": 113, "n_tasks_passed_any": 95, "ci_passed": 304, "ci_attempted": 436, "ci_lo": 0.656912963913939, "ci_hi": 0.7375824489300977, "ci_half": 0.04033474250807936, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 21.634702092592594, "median_cost_usd": 19.23457225, "mean_output_tokens": 118592.77293577982, "median_output_tokens": 115631.0, "mean_input_tokens": 12638832.12614679, "median_input_tokens": 9769437.5, "mean_cache_tokens": 12388674.334862385, "mean_duration_seconds": 2092.0106240022938, "median_duration_seconds": 1895.921464, "mean_agent_steps": 88.4288990825688, "median_agent_steps": 79.0, "median_peak_context_tokens": 201598.0, "median_output_tokens_to_pass": 115551.0 }, { "model": "gpt-5-6-terra", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_gpt_5_6_terra_max", "source": "deep-swe", "pass_rate": 0.6962305986696231, "pass_at_1": 0.6962305986696231, "pass_at_4": 0.8849557522123894, "n_passed": 314, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 100, "ci_passed": 314, "ci_attempted": 451, "ci_lo": 0.6706612091651647, "ci_hi": 0.7217999881740815, "ci_half": 0.025569389504458404, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.945847263858093, "median_cost_usd": 4.114557, "mean_output_tokens": 71938.62527716186, "median_output_tokens": 70835.0, "mean_input_tokens": 9230560.541019956, "median_input_tokens": 7288778.0, "mean_cache_tokens": 8652201.720620843, "mean_duration_seconds": 1016.6698429312638, "median_duration_seconds": 927.549964, "mean_agent_steps": 75.9290465631929, "median_agent_steps": 71.0, "median_peak_context_tokens": 177398.0, "median_output_tokens_to_pass": 70061.5 }, { "model": "gpt-5-6-sol", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gpt_5_6_sol_high", "source": "deep-swe", "pass_rate": 0.6940133037694013, "pass_at_1": 0.6940133037694013, "pass_at_4": 0.8672566371681416, "n_passed": 313, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 98, "ci_passed": 313, "ci_attempted": 451, "ci_lo": 0.679691851565629, "ci_hi": 0.7083347559731736, "ci_half": 0.014321452203772355, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.4698331108647453, "median_cost_usd": 2.979421, "mean_output_tokens": 28450.31929046563, "median_output_tokens": 27700.0, "mean_input_tokens": 2713572.2505543237, "median_input_tokens": 1979416.0, "mean_cache_tokens": 2435251.370288248, "mean_duration_seconds": 594.230510789357, "median_duration_seconds": 517.298681, "mean_agent_steps": 36.889135254988915, "median_agent_steps": 32.0, "median_peak_context_tokens": 98355.0, "median_output_tokens_to_pass": 27706.0 }, { "model": "glm-5-3", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_glm_5_3_max", "source": "deep-swe", "pass_rate": 0.6895787139689579, "pass_at_1": 0.6895787139689579, "pass_at_4": 0.8761061946902655, "n_passed": 311, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 99, "ci_passed": 311, "ci_attempted": 451, "ci_lo": 0.6593922012533708, "ci_hi": 0.7197652266845449, "ci_half": 0.030186512715587, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.9933584893126386, "median_cost_usd": 3.1315945999999992, "mean_output_tokens": 80435.60975609756, "median_output_tokens": 74953.0, "mean_input_tokens": 13272344.266075388, "median_input_tokens": 10137783.0, "mean_cache_tokens": 13106008.833702883, "mean_duration_seconds": 2116.1784804168515, "median_duration_seconds": 1842.82226, "mean_agent_steps": 124.47228381374723, "median_agent_steps": 114.0, "median_peak_context_tokens": 144729.0, "median_output_tokens_to_pass": 74814.0 }, { "model": "claude-opus-5", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_claude_opus_5_medium", "source": "deep-swe", "pass_rate": 0.6890380313199105, "pass_at_1": 0.6890380313199105, "pass_at_4": 0.8938053097345132, "n_passed": 308, "n_attempted": 447, "n_tasks_attempted": 113, "n_tasks_passed_any": 101, "ci_passed": 308, "ci_attempted": 447, "ci_lo": 0.6773124446609795, "ci_hi": 0.7007636179788415, "ci_half": 0.011725586658930944, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.28978764541387, "median_cost_usd": 2.4819885000000004, "mean_output_tokens": 36981.530201342284, "median_output_tokens": 33436.0, "mean_input_tokens": 3579815.3601789707, "median_input_tokens": 2211638.0, "mean_cache_tokens": 3479530.087248322, "mean_duration_seconds": 759.2254874630873, "median_duration_seconds": 587.101605, "mean_agent_steps": 52.29530201342282, "median_agent_steps": 43.0, "median_peak_context_tokens": 78690.0, "median_output_tokens_to_pass": 33678.0 }, { "model": "claude-fable-5", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_claude_fable_5_high", "source": "deep-swe", "pass_rate": 0.686046511627907, "pass_at_1": 0.686046511627907, "pass_at_4": 0.8672566371681416, "n_passed": 295, "n_attempted": 430, "n_tasks_attempted": 113, "n_tasks_passed_any": 98, "ci_passed": 295, "ci_attempted": 430, "ci_lo": 0.674838084453603, "ci_hi": 0.6972549388022109, "ci_half": 0.011208427174303877, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 9.177638214788733, "median_cost_usd": 7.322686000000002, "mean_output_tokens": 57287.06976744186, "median_output_tokens": 52623.5, "mean_input_tokens": 4833178.365116279, "median_input_tokens": 3075198.5, "mean_cache_tokens": 4711714.3162790695, "mean_duration_seconds": 1062.3037221116278, "median_duration_seconds": 902.496624, "mean_agent_steps": 58.73720930232558, "median_agent_steps": 49.0, "median_peak_context_tokens": 99524.0, "median_output_tokens_to_pass": 53715.0 }, { "model": "kimi-k3", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_kimi_k3_max", "source": "deep-swe", "pass_rate": 0.6851441241685144, "pass_at_1": 0.6851441241685144, "pass_at_4": 0.8938053097345132, "n_passed": 309, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 101, "ci_passed": 309, "ci_attempted": 451, "ci_lo": 0.6397739833756912, "ci_hi": 0.7305142649613376, "ci_half": 0.045370140792823206, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.654682129933482, "median_cost_usd": 3.6024987, "mean_output_tokens": 81499.84257206209, "median_output_tokens": 75383.0, "mean_input_tokens": 9695436.246119734, "median_input_tokens": 6930356.0, "mean_cache_tokens": 9501527.498891352, "mean_duration_seconds": 4541.447376412417, "median_duration_seconds": 3972.857633, "mean_agent_steps": 97.58758314855876, "median_agent_steps": 88.0, "median_peak_context_tokens": 130945.0, "median_output_tokens_to_pass": 74586.0 }, { "model": "grok-4-6", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_grok_4_6_medium", "source": "deep-swe", "pass_rate": 0.6747787610619469, "pass_at_1": 0.6747787610619469, "pass_at_4": 0.8407079646017699, "n_passed": 305, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 95, "ci_passed": 305, "ci_attempted": 452, "ci_lo": 0.6519707153331677, "ci_hi": 0.697586806790726, "ci_half": 0.0228080457287792, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.4489837522123894, "median_cost_usd": 2.945834, "mean_output_tokens": 49763.99778761062, "median_output_tokens": 48585.5, "mean_input_tokens": 5725195.139380531, "median_input_tokens": 4705124.0, "mean_cache_tokens": 5533327.0088495575, "mean_duration_seconds": 895.5380579646018, "median_duration_seconds": 860.0352045, "mean_agent_steps": 70.28982300884955, "median_agent_steps": 66.0, "median_peak_context_tokens": 116622.0, "median_output_tokens_to_pass": 48346.0 }, { "model": "gpt-5-6-luna", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_gpt_5_6_luna_max", "source": "deep-swe", "pass_rate": 0.671875, "pass_at_1": 0.671875, "pass_at_4": 0.9026548672566371, "n_passed": 301, "n_attempted": 448, "n_tasks_attempted": 113, "n_tasks_passed_any": 102, "ci_passed": 301, "ci_attempted": 448, "ci_lo": 0.6319549876432895, "ci_hi": 0.7117950123567105, "ci_half": 0.03992001235671055, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.028116810267857, "median_cost_usd": 2.2920917999999997, "mean_output_tokens": 73399.70758928571, "median_output_tokens": 70253.0, "mean_input_tokens": 15443718.330357144, "median_input_tokens": 12278493.0, "mean_cache_tokens": 14666893.714285715, "mean_duration_seconds": 1122.9470243995536, "median_duration_seconds": 982.9724034999999, "mean_agent_steps": 101.68080357142857, "median_agent_steps": 92.5, "median_peak_context_tokens": 201647.0, "median_output_tokens_to_pass": 68234.0 }, { "model": "gpt-5-5", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_5_5_xhigh", "source": "deep-swe", "pass_rate": 0.6703539823008849, "pass_at_1": 0.6703539823008849, "pass_at_4": 0.8849557522123894, "n_passed": 303, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 100, "ci_passed": 303, "ci_attempted": 452, "ci_lo": 0.6056975188917963, "ci_hi": 0.7350104457099735, "ci_half": 0.06465646340908864, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 7.226236674778761, "median_cost_usd": 6.110482499999999, "mean_output_tokens": 46294.72345132743, "median_output_tokens": 44492.5, "mean_input_tokens": 8372927.630530974, "median_input_tokens": 6648149.5, "mean_cache_tokens": 8032415.716814159, "mean_duration_seconds": 1806.3251869889382, "median_duration_seconds": 1588.3887945, "mean_agent_steps": 82.01548672566372, "median_agent_steps": 76.5, "median_peak_context_tokens": 141957.5, "median_output_tokens_to_pass": 44036.0 }, { "model": "gpt-6-astra", "harness": "mini-swe-agent", "provider": "openai", "reasoning_effort": "low", "config": "mini_swe_agent_gpt_6_astra_low", "source": "deep-swe", "cost_basis": "Expected launch pricing at all context lengths: $12/M uncached input, $15/M cache writes, $1.20/M cache reads, $50/M output, $2/M compute units.", "pass_rate": 0.6703539823008849, "pass_at_1": 0.6703539823008849, "pass_at_4": 0.7964601769911505, "n_passed": 303, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 90, "completed_by_attempt": [ 113, 113, 113, 113 ], "pass_rate_by_attempt": [ 0.6637168141592921, 0.6902654867256637, 0.6637168141592921, 0.6637168141592921 ], "ci_passed": 303, "ci_attempted": 452, "ci_lo": 0.6573453717840262, "ci_hi": 0.6833625928177437, "ci_half": 0.013008610516858731, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.188762706637168, "median_cost_usd": 1.7209886, "mean_output_tokens": 10579.53982300885, "median_output_tokens": 9487.5, "mean_input_tokens": 494506.91150442476, "median_input_tokens": 310994.0, "mean_uncached_input_tokens": 290.17035398230087, "median_uncached_input_tokens": 54.0, "mean_cache_tokens": 444727.3495575221, "mean_cache_read_tokens": 444727.3495575221, "median_cache_read_tokens": 269581.5, "mean_cache_write_tokens": 49489.39159292035, "median_cache_write_tokens": 37646.5, "mean_reasoning_tokens": 1414.641592920354, "median_reasoning_tokens": 848.5, "mean_compute_units": 190144.9889380531, "median_compute_units": 147460.0, "mean_duration_seconds": 613.4247787610619, "median_duration_seconds": 480.5, "mean_agent_steps": 19.537610619469028, "median_agent_steps": 17.0, "median_peak_context_tokens": null, "median_output_tokens_to_pass": 9650 }, { "model": "grok-4-6", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_grok_4_6_xhigh", "source": "deep-swe", "pass_rate": 0.6674057649667405, "pass_at_1": 0.6674057649667405, "pass_at_4": 0.8495575221238938, "n_passed": 301, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 96, "ci_passed": 301, "ci_attempted": 451, "ci_lo": 0.64564892272269, "ci_hi": 0.689162607210791, "ci_half": 0.02175684224405049, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 5.497668745011087, "median_cost_usd": 4.63937, "mean_output_tokens": 71403.54323725056, "median_output_tokens": 71285.0, "mean_input_tokens": 9344022.093126386, "median_input_tokens": 7667197.0, "mean_cache_tokens": 9079197.80044346, "mean_duration_seconds": 1272.3478393858093, "median_duration_seconds": 1245.896303, "mean_agent_steps": 87.21951219512195, "median_agent_steps": 81.0, "median_peak_context_tokens": 156581.0, "median_output_tokens_to_pass": 70062.0 }, { "model": "gemini-3-7-flash", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gemini_3_7_flash_medium", "source": "deep-swe", "pass_rate": 0.6548672566371682, "pass_at_1": 0.6548672566371682, "pass_at_4": 0.831858407079646, "n_passed": 296, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 94, "ci_passed": 296, "ci_attempted": 452, "ci_lo": 0.624001933873013, "ci_hi": 0.6857325794013234, "ci_half": 0.030865322764155222, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.0251351826880533, "median_cost_usd": 1.65422055, "mean_output_tokens": 93990.8517699115, "median_output_tokens": 87640.5, "mean_input_tokens": 15332065.907079646, "median_input_tokens": 11116142.0, "mean_cache_tokens": 14557599.913716814, "mean_duration_seconds": 1240.376194079646, "median_duration_seconds": 1066.1105834999998, "mean_agent_steps": 117.36946902654867, "median_agent_steps": 108.0, "median_peak_context_tokens": 195705.5, "median_output_tokens_to_pass": 87132.5 }, { "model": "claude-fable-5", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_claude_fable_5_medium", "source": "deep-swe", "pass_rate": 0.6536697247706422, "pass_at_1": 0.6536697247706422, "pass_at_4": 0.831858407079646, "n_passed": 285, "n_attempted": 436, "n_tasks_attempted": 113, "n_tasks_passed_any": 94, "ci_passed": 285, "ci_attempted": 436, "ci_lo": 0.6094504359782605, "ci_hi": 0.6978890135630239, "ci_half": 0.044219288792381656, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 6.088186528935186, "median_cost_usd": 4.62221125, "mean_output_tokens": 40201.35321100918, "median_output_tokens": 35970.5, "mean_input_tokens": 2942251.607798165, "median_input_tokens": 1732620.5, "mean_cache_tokens": 2848228.9174311925, "mean_duration_seconds": 810.8967150733945, "median_duration_seconds": 639.0384615, "mean_agent_steps": 48.36697247706422, "median_agent_steps": 41.0, "median_peak_context_tokens": 71989.5, "median_output_tokens_to_pass": 35214.0 }, { "model": "gemini-3-7-flash", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gemini_3_7_flash_high", "source": "deep-swe", "pass_rate": 0.6526548672566371, "pass_at_1": 0.6526548672566371, "pass_at_4": 0.8230088495575221, "n_passed": 295, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 93, "ci_passed": 295, "ci_attempted": 452, "ci_lo": 0.6347762421887939, "ci_hi": 0.6705334923244803, "ci_half": 0.017878625067843174, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.1762860351216813, "median_cost_usd": 1.9044440625, "mean_output_tokens": 107248.30309734514, "median_output_tokens": 99819.0, "mean_input_tokens": 16999853.159292035, "median_input_tokens": 13967881.5, "mean_cache_tokens": 16260422.17920354, "mean_duration_seconds": 1244.9565387986725, "median_duration_seconds": 1064.5330275000001, "mean_agent_steps": 124.51548672566372, "median_agent_steps": 120.0, "median_peak_context_tokens": 221180.5, "median_output_tokens_to_pass": 101363.0 }, { "model": "grok-4-6", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_grok_4_6_high", "source": "deep-swe", "pass_rate": 0.6518847006651884, "pass_at_1": 0.6518847006651884, "pass_at_4": 0.8495575221238938, "n_passed": 294, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 96, "ci_passed": 294, "ci_attempted": 451, "ci_lo": 0.6365486924786155, "ci_hi": 0.6672207088517614, "ci_half": 0.015336008186572951, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.3848732239467845, "median_cost_usd": 3.693604, "mean_output_tokens": 61160.94456762749, "median_output_tokens": 60060.0, "mean_input_tokens": 7348488.252771619, "median_input_tokens": 6003946.0, "mean_cache_tokens": 7119379.299334812, "mean_duration_seconds": 1083.6365204833703, "median_duration_seconds": 1031.198421, "mean_agent_steps": 78.960088691796, "median_agent_steps": 72.0, "median_peak_context_tokens": 136248.0, "median_output_tokens_to_pass": 59255.5 }, { "model": "gpt-5-5", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gpt_5_5_high", "source": "deep-swe", "pass_rate": 0.6438053097345132, "pass_at_1": 0.6438053097345132, "pass_at_4": 0.9026548672566371, "n_passed": 291, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 102, "ci_passed": 291, "ci_attempted": 452, "ci_lo": 0.6126368832394585, "ci_hi": 0.6749737362295679, "ci_half": 0.031168426495054677, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 5.100437004424779, "median_cost_usd": 4.582879500000001, "mean_output_tokens": 31159.49778761062, "median_output_tokens": 30377.5, "mean_input_tokens": 4617781.876106195, "median_input_tokens": 3860162.0, "mean_cache_tokens": 4205870.442477876, "mean_duration_seconds": 2048.0589126172567, "median_duration_seconds": 1881.0672625, "mean_agent_steps": 61.924778761061944, "median_agent_steps": 60.0, "median_peak_context_tokens": 107945.5, "median_output_tokens_to_pass": 30122.0 }, { "model": "glm-5-3-flash", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_glm_5_3_flash_max", "source": "deep-swe", "pass_rate": 0.6339285714285714, "pass_at_1": 0.6339285714285714, "pass_at_4": 0.8495575221238938, "n_passed": 284, "n_attempted": 448, "n_tasks_attempted": 113, "n_tasks_passed_any": 96, "ci_passed": 284, "ci_attempted": 448, "ci_lo": 0.590148863911635, "ci_hi": 0.6777082789455078, "ci_half": 0.04377970751693644, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.48196371245535713, "median_cost_usd": 0.389975125, "mean_output_tokens": 72829.77008928571, "median_output_tokens": 67491.0, "mean_input_tokens": 12387025.401785715, "median_input_tokens": 9358476.5, "mean_cache_tokens": 11770874.857142856, "mean_duration_seconds": 1537.7995419129463, "median_duration_seconds": 1322.635727, "mean_agent_steps": 122.890625, "median_agent_steps": 110.0, "median_peak_context_tokens": 136183.5, "median_output_tokens_to_pass": 68067.5 }, { "model": "deepseek-v4-pro", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_deepseek_v4_pro_max", "source": "deep-swe", "pass_rate": 0.6283185840707964, "pass_at_1": 0.6283185840707964, "pass_at_4": 0.8849557522123894, "n_passed": 284, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 100, "ci_passed": 284, "ci_attempted": 452, "ci_lo": 0.5649842780985077, "ci_hi": 0.6916528900430852, "ci_half": 0.06333430597228876, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.24138687892256638, "median_cost_usd": 0.2232531505000001, "mean_output_tokens": 105998.91814159292, "median_output_tokens": 101213.5, "mean_input_tokens": 24191606.099557523, "median_input_tokens": 20628260.5, "mean_cache_tokens": 24049100.743362833, "mean_duration_seconds": 2212.510104011062, "median_duration_seconds": 2104.8520015000004, "mean_agent_steps": 154.71238938053096, "median_agent_steps": 146.0, "median_peak_context_tokens": 232374.5, "median_output_tokens_to_pass": 100505.5 }, { "model": "gpt-5-6-sol", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gpt_5_6_sol_medium", "source": "deep-swe", "pass_rate": 0.6106194690265486, "pass_at_1": 0.6106194690265486, "pass_at_4": 0.8053097345132744, "n_passed": 276, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "ci_passed": 276, "ci_attempted": 452, "ci_lo": 0.5947858925334765, "ci_hi": 0.6264530455196208, "ci_half": 0.01583357649307215, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.8620318075221238, "median_cost_usd": 1.545414, "mean_output_tokens": 18425.216814159292, "median_output_tokens": 17645.0, "mean_input_tokens": 1505793.9557522123, "median_input_tokens": 1064552.5, "mean_cache_tokens": 1382962.9734513275, "mean_duration_seconds": 423.3752944181416, "median_duration_seconds": 354.895511, "mean_agent_steps": 30.913716814159294, "median_agent_steps": 26.0, "median_peak_context_tokens": 65185.0, "median_output_tokens_to_pass": 17452.0 }, { "model": "gpt-5-6-terra", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_5_6_terra_xhigh", "source": "deep-swe", "pass_rate": 0.6017699115044248, "pass_at_1": 0.6017699115044248, "pass_at_4": 0.8053097345132744, "n_passed": 272, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "ci_passed": 272, "ci_attempted": 452, "ci_lo": 0.5805269394851535, "ci_hi": 0.6230128835236961, "ci_half": 0.021242972019271257, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.127191725663717, "median_cost_usd": 1.8670102499999999, "mean_output_tokens": 39616.53982300885, "median_output_tokens": 38006.5, "mean_input_tokens": 3246355.4668141594, "median_input_tokens": 2490234.5, "mean_cache_tokens": 2926653.1681415928, "mean_duration_seconds": 584.0832282876107, "median_duration_seconds": 509.04962550000005, "mean_agent_steps": 43.06637168141593, "median_agent_steps": 39.0, "median_peak_context_tokens": 108741.0, "median_output_tokens_to_pass": 37940.5 }, { "model": "claude-fable-5", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_claude_fable_5_low", "source": "deep-swe", "pass_rate": 0.5958429561200924, "pass_at_1": 0.5958429561200924, "pass_at_4": 0.8141592920353983, "n_passed": 258, "n_attempted": 433, "n_tasks_attempted": 113, "n_tasks_passed_any": 92, "ci_passed": 258, "ci_attempted": 433, "ci_lo": 0.5679029016687951, "ci_hi": 0.6237830105713897, "ci_half": 0.027940054451297294, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.757873125874126, "median_cost_usd": 2.7559204999999998, "mean_output_tokens": 25242.833718244805, "median_output_tokens": 22339.0, "mean_input_tokens": 1691250.452655889, "median_input_tokens": 815862.0, "mean_cache_tokens": 1624197.8868360277, "mean_duration_seconds": 634.7057336951501, "median_duration_seconds": 397.628238, "mean_agent_steps": 37.79907621247113, "median_agent_steps": 29.0, "median_peak_context_tokens": 48670.0, "median_output_tokens_to_pass": 21899.5 }, { "model": "claude-opus-4-8", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_claude_opus_4_8_max", "source": "deep-swe", "pass_rate": 0.5897435897435898, "pass_at_1": 0.5897435897435898, "pass_at_4": 0.7927927927927928, "n_passed": 253, "n_attempted": 429, "n_tasks_attempted": 111, "n_tasks_passed_any": 88, "ci_passed": 253, "ci_attempted": 429, "ci_lo": 0.5720954362692973, "ci_hi": 0.6073917432178823, "ci_half": 0.01764815347429254, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 13.222583593240094, "median_cost_usd": 12.353505999999996, "mean_output_tokens": 135031.67132867133, "median_output_tokens": 129180.0, "mean_input_tokens": 17047938.554778554, "median_input_tokens": 15368311.0, "mean_cache_tokens": 16817766.494172495, "mean_duration_seconds": 3492.740812641026, "median_duration_seconds": 2883.384473, "mean_agent_steps": 120.0, "median_agent_steps": 116.0, "median_peak_context_tokens": 225741.0, "median_output_tokens_to_pass": 125257.0 }, { "model": "claude-opus-5", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_claude_opus_5_low", "source": "deep-swe", "pass_rate": 0.5812917594654788, "pass_at_1": 0.5812917594654788, "pass_at_4": 0.8495575221238938, "n_passed": 261, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 96, "ci_passed": 261, "ci_attempted": 449, "ci_lo": 0.5579848799986361, "ci_hi": 0.6045986389323216, "ci_half": 0.023306879466842723, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.6626223919821828, "median_cost_usd": 1.2162672499999996, "mean_output_tokens": 19884.3140311804, "median_output_tokens": 17613.0, "mean_input_tokens": 1564287.6570155902, "median_input_tokens": 872162.0, "mean_cache_tokens": 1497598.9933184856, "mean_duration_seconds": 468.87542417149217, "median_duration_seconds": 333.286568, "mean_agent_steps": 35.641425389755014, "median_agent_steps": 29.0, "median_peak_context_tokens": 46865.0, "median_output_tokens_to_pass": 17502.0 }, { "model": "qwen3-8-max", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_qwen3_8_max_xhigh", "source": "deep-swe", "pass_rate": 0.5746102449888641, "pass_at_1": 0.5746102449888641, "pass_at_4": 0.831858407079646, "n_passed": 258, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 94, "ci_passed": 258, "ci_attempted": 449, "ci_lo": 0.5479823182637887, "ci_hi": 0.6012381717139396, "ci_half": 0.026627926725075454, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.7290646500445437, "median_cost_usd": 3.132689060000001, "mean_output_tokens": 95075.18262806236, "median_output_tokens": 89729.0, "mean_input_tokens": 12996066.461024499, "median_input_tokens": 10163801.0, "mean_cache_tokens": 12477251.240534522, "mean_duration_seconds": 2567.2746634654786, "median_duration_seconds": 2424.198128, "mean_agent_steps": 111.33853006681514, "median_agent_steps": 102.0, "median_peak_context_tokens": 160572.0, "median_output_tokens_to_pass": 91083.0 }, { "model": "gpt-5-6-luna", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_5_6_luna_xhigh", "source": "deep-swe", "pass_rate": 0.5685840707964602, "pass_at_1": 0.5685840707964602, "pass_at_4": 0.7876106194690266, "n_passed": 257, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 89, "ci_passed": 257, "ci_attempted": 452, "ci_lo": 0.5469030532683623, "ci_hi": 0.5902650883245582, "ci_half": 0.021681017528097975, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.5356192685840708, "median_cost_usd": 1.2449332, "mean_output_tokens": 44677.900442477876, "median_output_tokens": 42374.5, "mean_input_tokens": 7610577.53539823, "median_input_tokens": 5550213.0, "mean_cache_tokens": 7096152.3539823005, "mean_duration_seconds": 731.6846703871681, "median_duration_seconds": 608.986619, "mean_agent_steps": 71.09955752212389, "median_agent_steps": 63.0, "median_peak_context_tokens": 138126.5, "median_output_tokens_to_pass": 41546.0 }, { "model": "muse-spark-1-2", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_muse_spark_1_2_xhigh", "source": "deep-swe", "pass_rate": 0.5486725663716814, "pass_at_1": 0.5486725663716814, "pass_at_4": 0.8141592920353983, "n_passed": 248, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 92, "ci_passed": 248, "ci_attempted": 452, "ci_lo": 0.5274295943524101, "ci_hi": 0.5699155383909527, "ci_half": 0.02124297201927127, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.6955048180309733, "median_cost_usd": 2.6840506, "mean_output_tokens": 99226.38053097345, "median_output_tokens": 81250.0, "mean_input_tokens": 13693824.435840707, "median_input_tokens": 10339991.0, "mean_cache_tokens": 12584988.949115044, "mean_duration_seconds": 927.8692516349557, "median_duration_seconds": 784.787489, "mean_agent_steps": 100.75884955752213, "median_agent_steps": 94.0, "median_peak_context_tokens": 189481.0, "median_output_tokens_to_pass": 77863.0 }, { "model": "claude-opus-4-8", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_claude_opus_4_8_xhigh", "source": "deep-swe", "pass_rate": 0.5436241610738255, "pass_at_1": 0.5436241610738255, "pass_at_4": 0.8053097345132744, "n_passed": 243, "n_attempted": 447, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "ci_passed": 243, "ci_attempted": 447, "ci_lo": 0.5064898547928808, "ci_hi": 0.5807584673547702, "ci_half": 0.03713430628094476, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 8.006355704138702, "median_cost_usd": 7.049357750000001, "mean_output_tokens": 86088.7874720358, "median_output_tokens": 77961.0, "mean_input_tokens": 9854652.170022372, "median_input_tokens": 8117323.0, "mean_cache_tokens": 9693014.304250559, "mean_duration_seconds": 2198.774075261745, "median_duration_seconds": 1829.190182, "mean_agent_steps": 94.64429530201342, "median_agent_steps": 90.0, "median_peak_context_tokens": 153870.0, "median_output_tokens_to_pass": 76018.0 }, { "model": "gpt-5-5", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gpt_5_5_medium", "source": "deep-swe", "pass_rate": 0.5398230088495575, "pass_at_1": 0.5398230088495575, "pass_at_4": 0.7787610619469026, "n_passed": 244, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 88, "ci_passed": 244, "ci_attempted": 452, "ci_lo": 0.5142921338966476, "ci_hi": 0.5653538838024673, "ci_half": 0.02553087495290979, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.749342734513274, "median_cost_usd": 2.436942, "mean_output_tokens": 19625.433628318584, "median_output_tokens": 18737.0, "mean_input_tokens": 2438539.053097345, "median_input_tokens": 1983652.5, "mean_cache_tokens": 2229359.0088495575, "mean_duration_seconds": 1288.995063800885, "median_duration_seconds": 1132.512228, "mean_agent_steps": 45.982300884955755, "median_agent_steps": 43.0, "median_peak_context_tokens": 75466.0, "median_output_tokens_to_pass": 18448.5 }, { "model": "claude-sonnet-5", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_claude_sonnet_5_max", "source": "deep-swe", "pass_rate": 0.5384615384615384, "pass_at_1": 0.5384615384615384, "pass_at_4": 0.7876106194690266, "n_passed": 238, "n_attempted": 442, "n_tasks_attempted": 113, "n_tasks_passed_any": 89, "ci_passed": 238, "ci_attempted": 442, "ci_lo": 0.4960923737597946, "ci_hi": 0.5808307031632823, "ci_half": 0.04236916470174386, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 26.399858950791852, "median_cost_usd": 23.277376725000003, "mean_output_tokens": 214117.54977375566, "median_output_tokens": 203917.5, "mean_input_tokens": 72415756.48642534, "median_input_tokens": 62591138.0, "mean_cache_tokens": 71991474.90950227, "mean_duration_seconds": 4803.543157789592, "median_duration_seconds": 4345.498858000001, "mean_agent_steps": 268.4547511312217, "median_agent_steps": 260.0, "median_peak_context_tokens": 409978.5, "median_output_tokens_to_pass": 203255.5 }, { "model": "gemini-3-7-flash", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_gemini_3_7_flash_low", "source": "deep-swe", "pass_rate": 0.5376106194690266, "pass_at_1": 0.5376106194690266, "pass_at_4": 0.7787610619469026, "n_passed": 243, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 88, "ci_passed": 243, "ci_attempted": 452, "ci_lo": 0.5117141286507079, "ci_hi": 0.5635071102873452, "ci_half": 0.02589649081831869, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.8322645629424779, "median_cost_usd": 1.3913454375, "mean_output_tokens": 73364.9557522124, "median_output_tokens": 73220.0, "mean_input_tokens": 15056565.075221239, "median_input_tokens": 9612558.5, "mean_cache_tokens": 14422633.818584071, "mean_duration_seconds": 1252.0731730707964, "median_duration_seconds": 1102.7230410000002, "mean_agent_steps": 130.37610619469027, "median_agent_steps": 113.5, "median_peak_context_tokens": 168146.0, "median_output_tokens_to_pass": 75057.0 }, { "model": "gpt-5-6-terra", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gpt_5_6_terra_high", "source": "deep-swe", "pass_rate": 0.5376106194690266, "pass_at_1": 0.5376106194690266, "pass_at_4": 0.8053097345132744, "n_passed": 243, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "ci_passed": 243, "ci_attempted": 452, "ci_lo": 0.49432091479689105, "ci_hi": 0.5809003241411621, "ci_half": 0.04328970467213552, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.1343762765486725, "median_cost_usd": 1.031709, "mean_output_tokens": 21517.03982300885, "median_output_tokens": 20865.5, "mean_input_tokens": 1557052.774336283, "median_input_tokens": 1249365.5, "mean_cache_tokens": 1369338.3362831858, "mean_duration_seconds": 364.2859819269911, "median_duration_seconds": 314.963429, "mean_agent_steps": 33.51327433628319, "median_agent_steps": 31.0, "median_peak_context_tokens": 68750.0, "median_output_tokens_to_pass": 20408.0 }, { "model": "grok-4-5", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_grok_4_5_high", "source": "deep-swe", "pass_rate": 0.5376106194690266, "pass_at_1": 0.5376106194690266, "pass_at_4": 0.7787610619469026, "n_passed": 243, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 88, "ci_passed": 243, "ci_attempted": 452, "ci_lo": 0.5148025737402474, "ci_hi": 0.5604186651978057, "ci_half": 0.0228080457287792, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.4157481725663716, "median_cost_usd": 2.02311, "mean_output_tokens": 35525.33185840708, "median_output_tokens": 34250.0, "mean_input_tokens": 4059182.975663717, "median_input_tokens": 3293744.5, "mean_cache_tokens": 3943846.5132743362, "mean_duration_seconds": 484.7029359800885, "median_duration_seconds": 430.0852805, "mean_agent_steps": 61.33185840707964, "median_agent_steps": 56.0, "median_peak_context_tokens": 94044.0, "median_output_tokens_to_pass": 33872.0 }, { "model": "deepseek-v4-flash", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_deepseek_v4_flash_max", "source": "deep-swe", "pass_rate": 0.5331858407079646, "pass_at_1": 0.5331858407079646, "pass_at_4": 0.8053097345132744, "n_passed": 241, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 91, "ci_passed": 241, "ci_attempted": 452, "ci_lo": 0.4975163385576403, "ci_hi": 0.5688553428582889, "ci_half": 0.035669502150324287, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.10023560370265487, "median_cost_usd": 0.09124898719999996, "mean_output_tokens": 107687.02212389381, "median_output_tokens": 104491.5, "mean_input_tokens": 19829931.451327432, "median_input_tokens": 17312559.5, "mean_cache_tokens": 19719518.867256638, "mean_duration_seconds": 1438.9961937522125, "median_duration_seconds": 1383.7182735000001, "mean_agent_steps": 152.87610619469027, "median_agent_steps": 148.0, "median_peak_context_tokens": 201636.5, "median_output_tokens_to_pass": 100483.0 }, { "model": "muse-spark-1-1", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_muse_spark_1_1_xhigh", "source": "deep-swe", "pass_rate": 0.5331858407079646, "pass_at_1": 0.5331858407079646, "pass_at_4": 0.7964601769911505, "n_passed": 241, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 90, "ci_passed": 241, "ci_attempted": 452, "ci_lo": 0.5028324161686275, "ci_hi": 0.5635392652473017, "ci_half": 0.030353424539337114, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.361142644469026, "median_cost_usd": 1.7171158, "mean_output_tokens": 74008.4203539823, "median_output_tokens": 66088.0, "mean_input_tokens": 12005399.261061948, "median_input_tokens": 8037064.5, "mean_cache_tokens": 11781947.471238937, "mean_duration_seconds": 897.5133296747788, "median_duration_seconds": 688.5861825, "mean_agent_steps": 95.82522123893806, "median_agent_steps": 86.0, "median_peak_context_tokens": 154817.5, "median_output_tokens_to_pass": 61398.0 }, { "model": "claude-opus-4-8", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_claude_opus_4_8_high", "source": "deep-swe", "pass_rate": 0.5176991150442478, "pass_at_1": 0.5176991150442478, "pass_at_4": 0.7787610619469026, "n_passed": 234, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 88, "ci_passed": 234, "ci_attempted": 452, "ci_lo": 0.4720830235866894, "ci_hi": 0.5633152065018062, "ci_half": 0.04561609145755843, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.281940017699115, "median_cost_usd": 3.6635345, "mean_output_tokens": 50063.52654867257, "median_output_tokens": 44476.0, "mean_input_tokens": 4908687.0265486725, "median_input_tokens": 3814284.0, "mean_cache_tokens": 4807501.1283185845, "mean_duration_seconds": 1066.903783732301, "median_duration_seconds": 887.7987455, "mean_agent_steps": 72.5, "median_agent_steps": 67.0, "median_peak_context_tokens": 94158.5, "median_output_tokens_to_pass": 43664.0 }, { "model": "gpt-5-4", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_gpt_5_4_xhigh", "source": "deep-swe", "pass_rate": 0.5176991150442478, "pass_at_1": 0.5176991150442478, "pass_at_4": 0.7787610619469026, "n_passed": 234, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 88, "ci_passed": 234, "ci_attempted": 452, "ci_lo": 0.502678065476865, "ci_hi": 0.5327201646116306, "ci_half": 0.015021049567382833, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 5.6524577245575225, "median_cost_usd": 4.3528565, "mean_output_tokens": 71408.87389380531, "median_output_tokens": 67233.5, "mean_input_tokens": 9418920.763274336, "median_input_tokens": 6869950.5, "mean_cache_tokens": 8613618.407079646, "mean_duration_seconds": 1368.707164818584, "median_duration_seconds": 1209.517342, "mean_agent_steps": 70.47123893805309, "median_agent_steps": 63.0, "median_peak_context_tokens": 172734.5, "median_output_tokens_to_pass": 64347.0 }, { "model": "claude-sonnet-5", "harness": "mini-swe-agent", "reasoning_effort": "xhigh", "config": "mini_swe_agent_claude_sonnet_5_xhigh", "source": "deep-swe", "pass_rate": 0.49667405764966743, "pass_at_1": 0.49667405764966743, "pass_at_4": 0.7522123893805309, "n_passed": 224, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 85, "ci_passed": 224, "ci_attempted": 451, "ci_lo": 0.4621285677328888, "ci_hi": 0.5312195475664461, "ci_half": 0.034545489916778645, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 11.890634540022173, "median_cost_usd": 9.965361750000001, "mean_output_tokens": 120698.59201773835, "median_output_tokens": 113150.0, "mean_input_tokens": 30695627.87804878, "median_input_tokens": 24771332.0, "mean_cache_tokens": 30442947.99556541, "mean_duration_seconds": 2414.4529098159646, "median_duration_seconds": 2229.375982, "mean_agent_steps": 185.53215077605321, "median_agent_steps": 174.0, "median_peak_context_tokens": 240780.0, "median_output_tokens_to_pass": 112015.5 }, { "model": "claude-opus-4-8", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_claude_opus_4_8_medium", "source": "deep-swe", "pass_rate": 0.48672566371681414, "pass_at_1": 0.48672566371681414, "pass_at_4": 0.7610619469026548, "n_passed": 220, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 86, "ci_passed": 220, "ci_attempted": 452, "ci_lo": 0.4643336050994396, "ci_hi": 0.5091177223341886, "ci_half": 0.02239205861737452, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.4438571028761062, "median_cost_usd": 2.84043425, "mean_output_tokens": 41313.45132743363, "median_output_tokens": 36647.5, "mean_input_tokens": 3843602.6681415928, "median_input_tokens": 2851276.5, "mean_cache_tokens": 3757520.9115044246, "mean_duration_seconds": 880.0083837986726, "median_duration_seconds": 729.1386, "mean_agent_steps": 65.57300884955752, "median_agent_steps": 60.0, "median_peak_context_tokens": 79616.0, "median_output_tokens_to_pass": 36116.5 }, { "model": "claude-sonnet-5", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_claude_sonnet_5_high", "source": "deep-swe", "pass_rate": 0.4823008849557522, "pass_at_1": 0.4823008849557522, "pass_at_4": 0.7964601769911505, "n_passed": 218, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 90, "ci_passed": 218, "ci_attempted": 452, "ci_lo": 0.4372377362536038, "ci_hi": 0.5273640336579006, "ci_half": 0.045063148702148406, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 7.425560161197339, "median_cost_usd": 5.833649100000001, "mean_output_tokens": 87294.75609756098, "median_output_tokens": 77993.0, "mean_input_tokens": 18254491.34368071, "median_input_tokens": 13842595.0, "mean_cache_tokens": 18068889.902439024, "mean_duration_seconds": 1728.2193087920355, "median_duration_seconds": 1539.1866639999998, "mean_agent_steps": 146.57649667405764, "median_agent_steps": 138.0, "median_peak_context_tokens": 174563.0, "median_output_tokens_to_pass": 76281.0 }, { "model": "gemini-3-6-flash", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gemini_3_6_flash_high", "source": "deep-swe", "pass_rate": 0.4668141592920354, "pass_at_1": 0.4668141592920354, "pass_at_4": 0.7522123893805309, "n_passed": 211, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 85, "ci_passed": 211, "ci_attempted": 452, "ci_lo": 0.4297656202995626, "ci_hi": 0.5038626982845081, "ci_half": 0.037048538992472776, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.419013277333334, "median_cost_usd": 3.444695325000001, "mean_output_tokens": 95844.86222222222, "median_output_tokens": 86727.0, "mean_input_tokens": 12595957.344444444, "median_input_tokens": 8949018.5, "mean_cache_tokens": 11254636.448888889, "mean_duration_seconds": 1477.8607510707964, "median_duration_seconds": 1214.8670550000002, "mean_agent_steps": 116.73333333333333, "median_agent_steps": 108.0, "median_peak_context_tokens": 151398.5, "median_output_tokens_to_pass": 83414.0 }, { "model": "gpt-5-6-sol", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_gpt_5_6_sol_low", "source": "deep-swe", "pass_rate": 0.45353982300884954, "pass_at_1": 0.45353982300884954, "pass_at_4": 0.7168141592920354, "n_passed": 205, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 81, "ci_passed": 205, "ci_attempted": 452, "ci_lo": 0.42965787629426033, "ci_hi": 0.47742176972343875, "ci_half": 0.0238819467145892, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.0742664646017699, "median_cost_usd": 0.944557, "mean_output_tokens": 10579.141592920354, "median_output_tokens": 10062.5, "mean_input_tokens": 690833.657079646, "median_input_tokens": 541018.0, "mean_cache_tokens": 599908.814159292, "mean_duration_seconds": 262.2510164579646, "median_duration_seconds": 211.1408625, "mean_agent_steps": 23.358407079646017, "median_agent_steps": 21.0, "median_peak_context_tokens": 39823.0, "median_output_tokens_to_pass": 10031.0 }, { "model": "gpt-5-6-luna", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gpt_5_6_luna_high", "source": "deep-swe", "pass_rate": 0.4424778761061947, "pass_at_1": 0.4424778761061947, "pass_at_4": 0.7522123893805309, "n_passed": 200, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 85, "ci_passed": 200, "ci_attempted": 452, "ci_lo": 0.4132822036270294, "ci_hi": 0.47167354858536004, "ci_half": 0.029195672479165317, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.7779007499999999, "median_cost_usd": 0.6635542999999999, "mean_output_tokens": 25778.274336283186, "median_output_tokens": 24817.5, "mean_input_tokens": 3372477.7876106193, "median_input_tokens": 2538825.5, "mean_cache_tokens": 3055611.469026549, "mean_duration_seconds": 473.83880740929203, "median_duration_seconds": 394.07379549999996, "mean_agent_steps": 49.02212389380531, "median_agent_steps": 44.0, "median_peak_context_tokens": 90166.5, "median_output_tokens_to_pass": 24022.0 }, { "model": "glm-5-2", "harness": "mini-swe-agent", "reasoning_effort": "max", "config": "mini_swe_agent_glm_5_2_max", "source": "deep-swe", "pass_rate": 0.43777777777777777, "pass_at_1": 0.43777777777777777, "pass_at_4": 0.7699115044247787, "n_passed": 197, "n_attempted": 450, "n_tasks_attempted": 113, "n_tasks_passed_any": 87, "ci_passed": 197, "ci_attempted": 450, "ci_lo": 0.42052148157196967, "ci_hi": 0.45503407398358586, "ci_half": 0.01725629620580811, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.9199075743111114, "median_cost_usd": 3.4637479200000003, "mean_output_tokens": 78175.30666666667, "median_output_tokens": 75760.0, "mean_input_tokens": 12606560.215555556, "median_input_tokens": 11208642.0, "mean_cache_tokens": 12344954.453333333, "mean_duration_seconds": 2629.2083211311115, "median_duration_seconds": 2304.8350069999997, "mean_agent_steps": 129.12666666666667, "median_agent_steps": 123.0, "median_peak_context_tokens": 142129.0, "median_output_tokens_to_pass": 75146.0 }, { "model": "grok-4-6", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_grok_4_6_low", "source": "deep-swe", "pass_rate": 0.41648106904231624, "pass_at_1": 0.41648106904231624, "pass_at_4": 0.6902654867256637, "n_passed": 187, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 78, "ci_passed": 187, "ci_attempted": 449, "ci_lo": 0.3932945513980172, "ci_hi": 0.43966758668661526, "ci_half": 0.023186517644299014, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.0423604231625836, "median_cost_usd": 0.742614, "mean_output_tokens": 16458.3429844098, "median_output_tokens": 14937.0, "mean_input_tokens": 1646919.8017817372, "median_input_tokens": 1089649.0, "mean_cache_tokens": 1566819.4922048997, "mean_duration_seconds": 325.05958362360803, "median_duration_seconds": 273.420881, "mean_agent_steps": 44.18708240534521, "median_agent_steps": 39.0, "median_peak_context_tokens": 45300.0, "median_output_tokens_to_pass": 14781.0 }, { "model": "claude-opus-4-8", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_claude_opus_4_8_low", "source": "deep-swe", "pass_rate": 0.4079822616407982, "pass_at_1": 0.4079822616407982, "pass_at_4": 0.6814159292035398, "n_passed": 184, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 77, "ci_passed": 184, "ci_attempted": 451, "ci_lo": 0.3933464140551067, "ci_hi": 0.42261810922648974, "ci_half": 0.014635847585691517, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.293369216740577, "median_cost_usd": 1.8240152500000004, "mean_output_tokens": 28922.736141906873, "median_output_tokens": 25291.0, "mean_input_tokens": 2417987.955654102, "median_input_tokens": 1658113.0, "mean_cache_tokens": 2354415.691796009, "mean_duration_seconds": 651.8271712993347, "median_duration_seconds": 517.146142, "mean_agent_steps": 53.97782705099778, "median_agent_steps": 47.0, "median_peak_context_tokens": 57903.0, "median_output_tokens_to_pass": 24500.0 }, { "model": "claude-sonnet-5", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_claude_sonnet_5_medium", "source": "deep-swe", "pass_rate": 0.3977777777777778, "pass_at_1": 0.3977777777777778, "pass_at_4": 0.6460176991150443, "n_passed": 179, "n_attempted": 450, "n_tasks_attempted": 113, "n_tasks_passed_any": 73, "ci_passed": 179, "ci_attempted": 450, "ci_lo": 0.3665020157144805, "ci_hi": 0.4290535398410751, "ci_half": 0.03127576206329727, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 4.079036229, "median_cost_usd": 3.3573110999999995, "mean_output_tokens": 56816.753333333334, "median_output_tokens": 50414.5, "mean_input_tokens": 9286962.313333333, "median_input_tokens": 7084139.5, "mean_cache_tokens": 9158953.993333334, "mean_duration_seconds": 1121.270593551111, "median_duration_seconds": 980.8059874999999, "mean_agent_steps": 107.61111111111111, "median_agent_steps": 100.5, "median_peak_context_tokens": 118790.0, "median_output_tokens_to_pass": 49405.0 }, { "model": "glm-5-2", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_glm_5_2_high", "source": "deep-swe", "pass_rate": 0.36283185840707965, "pass_at_1": 0.36283185840707965, "pass_at_4": 0.6814159292035398, "n_passed": 164, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 77, "ci_passed": 164, "ci_attempted": 452, "ci_lo": 0.3153311289278631, "ci_hi": 0.41033258788629623, "ci_half": 0.047500729479216554, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.8355164725663715, "median_cost_usd": 2.25012998, "mean_output_tokens": 54245.50442477876, "median_output_tokens": 48956.5, "mean_input_tokens": 9070605.707964601, "median_input_tokens": 7049445.5, "mean_cache_tokens": 8861413.805309735, "mean_duration_seconds": 1794.4733617809734, "median_duration_seconds": 1565.2561315, "mean_agent_steps": 121.88274336283186, "median_agent_steps": 112.0, "median_peak_context_tokens": 102096.0, "median_output_tokens_to_pass": 47433.5 }, { "model": "gemini-3-5-flash", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gemini_3_5_flash_high", "source": "deep-swe", "pass_rate": 0.3606194690265487, "pass_at_1": 0.3606194690265487, "pass_at_4": 0.6371681415929203, "n_passed": 163, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 72, "ci_passed": 163, "ci_attempted": 452, "ci_lo": 0.3209564389213443, "ci_hi": 0.40028249913175307, "ci_half": 0.03966303010520439, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 3.4467114588691796, "median_cost_usd": 2.8624087500000015, "mean_output_tokens": 75730.19290465632, "median_output_tokens": 68910.0, "mean_input_tokens": 7629071.858093127, "median_input_tokens": 5694082.0, "mean_cache_tokens": 6428494.862527716, "mean_duration_seconds": 1125.1601607057523, "median_duration_seconds": 973.6770345, "mean_agent_steps": 105.30376940133037, "median_agent_steps": 97.0, "median_peak_context_tokens": 113979.0, "median_output_tokens_to_pass": 64913.0 }, { "model": "gpt-5-6-terra", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gpt_5_6_terra_medium", "source": "deep-swe", "pass_rate": 0.3511111111111111, "pass_at_1": 0.3511111111111111, "pass_at_4": 0.6017699115044248, "n_passed": 158, "n_attempted": 450, "n_tasks_attempted": 113, "n_tasks_passed_any": 68, "ci_passed": 158, "ci_attempted": 450, "ci_lo": 0.3172914680770448, "ci_hi": 0.3849307541451774, "ci_half": 0.03381964303406632, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.5832493688888889, "median_cost_usd": 0.546099, "mean_output_tokens": 11746.56, "median_output_tokens": 11453.0, "mean_input_tokens": 725100.8355555555, "median_input_tokens": 628495.0, "mean_cache_tokens": 624756.0533333333, "mean_duration_seconds": 229.60274282444442, "median_duration_seconds": 193.9660975, "mean_agent_steps": 25.14666666666667, "median_agent_steps": 24.0, "median_peak_context_tokens": 44178.0, "median_output_tokens_to_pass": 11448.0 }, { "model": "kimi-k2-7-code", "harness": "mini-swe-agent", "reasoning_effort": null, "config": "mini_swe_agent_kimi_k2_7_code_default", "source": "deep-swe", "pass_rate": 0.3053097345132743, "pass_at_1": 0.3053097345132743, "pass_at_4": 0.6106194690265486, "n_passed": 138, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 69, "ci_passed": 138, "ci_attempted": 452, "ci_lo": 0.3003027179908134, "ci_hi": 0.31031675103573525, "ci_half": 0.005007016522460913, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.815536243274336, "median_cost_usd": 2.1955079100000003, "mean_output_tokens": 59297.24778761062, "median_output_tokens": 54054.5, "mean_input_tokens": 13008041.482300885, "median_input_tokens": 10022682.0, "mean_cache_tokens": 12857596.451327434, "mean_duration_seconds": 2163.7950712809734, "median_duration_seconds": 1931.6495705, "mean_agent_steps": 149.1150442477876, "median_agent_steps": 139.0, "median_peak_context_tokens": 122628.5, "median_output_tokens_to_pass": 48384.0 }, { "model": "claude-sonnet-5", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_claude_sonnet_5_low", "source": "deep-swe", "pass_rate": 0.3051224944320713, "pass_at_1": 0.3051224944320713, "pass_at_4": 0.5752212389380531, "n_passed": 137, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 65, "ci_passed": 137, "ci_attempted": 449, "ci_lo": 0.2938623190176438, "ci_hi": 0.31638266984649877, "ci_half": 0.011260175414427497, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.1865550832962137, "median_cost_usd": 1.6764955500000003, "mean_output_tokens": 35595.08240534521, "median_output_tokens": 31478.0, "mean_input_tokens": 4534475.699331849, "median_input_tokens": 3126877.0, "mean_cache_tokens": 4449592.527839644, "mean_duration_seconds": 734.9105084298441, "median_duration_seconds": 624.639023, "mean_agent_steps": 76.88864142538975, "median_agent_steps": 70.0, "median_peak_context_tokens": 76944.0, "median_output_tokens_to_pass": 32131.0 }, { "model": "claude-sonnet-4-6", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_claude_sonnet_4_6_high", "source": "deep-swe", "pass_rate": 0.29933481152993346, "pass_at_1": 0.29933481152993346, "pass_at_4": 0.5663716814159292, "n_passed": 135, "n_attempted": 451, "n_tasks_attempted": 113, "n_tasks_passed_any": 64, "ci_passed": 135, "ci_attempted": 451, "ci_lo": 0.2584133988380421, "ci_hi": 0.34025622422182483, "ci_half": 0.04092141269189136, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 5.5223517987804875, "median_cost_usd": 4.867440150000001, "mean_output_tokens": 76160.31263858093, "median_output_tokens": 70001.0, "mean_input_tokens": 12870627.738359202, "median_input_tokens": 10933709.0, "mean_cache_tokens": 12719994.494456762, "mean_duration_seconds": 2815.01307943459, "median_duration_seconds": 2373.991057, "mean_agent_steps": 133.65853658536585, "median_agent_steps": 124.0, "median_peak_context_tokens": 146430.0, "median_output_tokens_to_pass": 67060.0 }, { "model": "gpt-5-5", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_gpt_5_5_low", "source": "deep-swe", "pass_rate": 0.26991150442477874, "pass_at_1": 0.26991150442477874, "pass_at_4": 0.4778761061946903, "n_passed": 122, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 54, "ci_passed": 122, "ci_attempted": 452, "ci_lo": 0.24696647220470694, "ci_hi": 0.29285653664485056, "ci_half": 0.02294503222007181, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 1.2002025442477877, "median_cost_usd": 1.108106, "mean_output_tokens": 9442.774336283186, "median_output_tokens": 9020.0, "mean_input_tokens": 776751.5840707965, "median_input_tokens": 645287.5, "mean_cache_tokens": 658930.407079646, "mean_duration_seconds": 569.9958436615044, "median_duration_seconds": 485.6083185, "mean_agent_steps": 28.06637168141593, "median_agent_steps": 27.0, "median_peak_context_tokens": 39278.0, "median_output_tokens_to_pass": 8910.5 }, { "model": "gpt-5-6-terra", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_gpt_5_6_terra_low", "source": "deep-swe", "pass_rate": 0.24053452115812918, "pass_at_1": 0.24053452115812918, "pass_at_4": 0.4424778761061947, "n_passed": 108, "n_attempted": 449, "n_tasks_attempted": 113, "n_tasks_passed_any": 50, "ci_passed": 108, "ci_attempted": 449, "ci_lo": 0.23276690748868134, "ci_hi": 0.24830213482757701, "ci_half": 0.007767613669447833, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.4277452605790646, "median_cost_usd": 0.3989895, "mean_output_tokens": 8572.26280623608, "median_output_tokens": 8265.0, "mean_input_tokens": 480565.34521158127, "median_input_tokens": 405397.0, "mean_cache_tokens": 401000.90868596884, "mean_duration_seconds": 176.52513473942093, "median_duration_seconds": 151.960538, "mean_agent_steps": 21.456570155902003, "median_agent_steps": 20.0, "median_peak_context_tokens": 33373.0, "median_output_tokens_to_pass": 8024.5 }, { "model": "gemini-3-1-pro-preview", "harness": "mini-swe-agent", "reasoning_effort": "high", "config": "mini_swe_agent_gemini_3_1_pro_preview_high", "source": "deep-swe", "pass_rate": 0.1172566371681416, "pass_at_1": 0.1172566371681416, "pass_at_4": 0.2831858407079646, "n_passed": 53, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 32, "ci_passed": 53, "ci_attempted": 452, "ci_lo": 0.10244568255705315, "ci_hi": 0.13206759177923005, "ci_half": 0.01481095461108845, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 2.1434045568888886, "median_cost_usd": 1.7235064000000002, "mean_output_tokens": 28368.884444444444, "median_output_tokens": 25990.5, "mean_input_tokens": 2403677.331111111, "median_input_tokens": 1834116.0, "mean_cache_tokens": 1678992.7311111111, "mean_duration_seconds": 867.7979107013274, "median_duration_seconds": 734.5636605, "mean_agent_steps": 75.56444444444445, "median_agent_steps": 70.0, "median_peak_context_tokens": 49745.5, "median_output_tokens_to_pass": 23839.0 }, { "model": "gpt-5-6-luna", "harness": "mini-swe-agent", "reasoning_effort": "medium", "config": "mini_swe_agent_gpt_5_6_luna_medium", "source": "deep-swe", "pass_rate": 0.11283185840707964, "pass_at_1": 0.11283185840707964, "pass_at_4": 0.2743362831858407, "n_passed": 51, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 31, "ci_passed": 51, "ci_attempted": 452, "ci_lo": 0.10452866084502313, "ci_hi": 0.12113505596913615, "ci_half": 0.008303197562056514, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.21630978982300886, "median_cost_usd": 0.1994702, "mean_output_tokens": 8179.570796460177, "median_output_tokens": 7918.5, "mean_input_tokens": 617827.4800884955, "median_input_tokens": 517201.0, "mean_cache_tokens": 500661.2389380531, "mean_duration_seconds": 181.5252541438053, "median_duration_seconds": 151.8156975, "mean_agent_steps": 23.67699115044248, "median_agent_steps": 22.0, "median_peak_context_tokens": 35812.0, "median_output_tokens_to_pass": 7414.0 }, { "model": "gpt-5-6-luna", "harness": "mini-swe-agent", "reasoning_effort": "low", "config": "mini_swe_agent_gpt_5_6_luna_low", "source": "deep-swe", "pass_rate": 0.015486725663716814, "pass_at_1": 0.015486725663716814, "pass_at_4": 0.04424778761061947, "n_passed": 7, "n_attempted": 452, "n_tasks_attempted": 113, "n_tasks_passed_any": 5, "ci_passed": 7, "ci_attempted": 452, "ci_lo": 0.0071835281016603, "ci_hi": 0.023789923225773328, "ci_half": 0.008303197562056514, "n_runs": 4, "ci_method": "95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))", "mean_cost_usd": 0.07240623938053097, "median_cost_usd": 0.0697851, "mean_output_tokens": 3127.754424778761, "median_output_tokens": 3059.0, "mean_input_tokens": 150031.52876106196, "median_input_tokens": 143435.0, "mean_cache_tokens": 107102.01769911505, "mean_duration_seconds": 77.14656007522123, "median_duration_seconds": 69.9232685, "mean_agent_steps": 12.464601769911505, "median_agent_steps": 12.0, "median_peak_context_tokens": 17490.5, "median_output_tokens_to_pass": 3036.0 } ] }