{ "name": "Fireworks Specialized Intelligence Index", "description": "Native benchmark scores, cost, and duration for open, closed, and specialized language models across domain benchmarks contributed by industry practitioners.", "url": "https://fireworks.ai/specialized-intelligence-index", "generated_at": "2026-09-22T20:04:14.944Z", "sources_checked": "sources checked Sep 21, 2026", "status": "published", "notice": null, "publisher": { "name": "Fireworks AI", "url": "https://fireworks.ai" }, "documentation": "https://fireworks.ai/specialized-intelligence-index/llms.txt", "formats": { "json": "https://fireworks.ai/specialized-intelligence-index/api/index.json", "markdown": "https://fireworks.ai/specialized-intelligence-index/index.md", "html": "https://fireworks.ai/specialized-intelligence-index/" }, "units": { "score": "native benchmark headline value; percentage scores use 0-100 and higher is better", "diagnosis_elo": "relative Elo points centered at 1000 within the evaluated model pool, higher is better", "cost_per_task": "US dollars per task, end to end including retries, lower is better", "cost_per_trial": "US dollars per trial, tested-model calls only, lower is better", "cost_per_deck": "US dollars per finished deck, lower is better", "duration_per_task": "wall-clock seconds per task, start to verifier finish, lower is better", "duration_per_trial": "wall-clock seconds for one trial of a repeated task, lower is better" }, "note_on_completeness": "Every metric published for every model is present in this file for every benchmark, including metrics the interactive site hides behind column toggles. A null value means the benchmark does not publish that metric, never that it was filtered out.", "methodology": { "formula": "Benchmark Index = 100 × A^0.706 × C^0.176 × L^0.118", "weights": { "score": 0.706, "cost": 0.176, "duration": 0.118 }, "anchor_low_quantile": 0.1, "anchor_high_quantile": 0.9, "minimum_benchmark_count": 1 }, "summary": [ "On dfbench, contributed by depthfirst, Mythos 5† (Anthropic) ranks first with 69% score (detection recall).", "On PWNBench v0.1, contributed by Novee, Grok 4.6 (xAI) ranks first with 63.31% score (F0.5), $330.65 per task.", "On CyberGym, DeepSeek V4.1 Flash (DeepSeek) ranks first with 94.82% score (pass@3), $6.53 per task, 1353 seconds per task.", "On LAB, contributed by Harvey, Grok 4.6 (xAI) ranks first with 22.08% score (all-pass), $4 per task, 2160 seconds per task.", "On LAB: Contracts, contributed by Harvey, Grok 4.6 (xAI) ranks first with 10% score (all-pass), $2.19 per task, 2160 seconds per task.", "On APEX Agents: Corporate Law, contributed by Mercor, GPT-6 Astra (OpenAI) ranks first with 87.67% score (avg@4), $4.42 per task.", "On Redline Bench, contributed by Crosby, GPT-6 Astra (max) (OpenAI) ranks first with 62.84% score (avg@3), $9.26 per task, 2286.4 seconds per task.", "On Big Finance Benchmark, contributed by Rogo, GPT-6 Astra (OpenAI) ranks first with 54.6% score (avg@3), $0.303 per task.", "On DuetBench–Diagnosis, contributed by Decagon, Claude Opus 5 (Anthropic) ranks first with 1034.9 score (Diagnosis Elo), $3.86 per task, 326.3 seconds per task.", "On τ³-Banking, contributed by Sierra, GLM-5.3 (Z.ai) ranks first with 51.63% score (mean trial reward), $0.4852 per trial, 281.6 seconds per trial.", "On τ³-Voice, contributed by Sierra, gpt-live-1 (OpenAI · backend: gpt-6 astra (medium) · v1.0) ranks first with 81.72% score (Pass@1), 2.5 seconds per task.", "On FrontierSWE V2, contributed by Proximal, GPT-6 Astra (OpenAI) ranks first with 65.5% score (mean@5), $5148.25 per task, 43560 seconds per trial.", "On APEX-SWE, contributed by Mercor, Claude Opus 5 (Anthropic) ranks first with 63.75% score (avg@4), $21.93 per task.", "On DeepSWE v1.1, Claude Opus 5 (Anthropic) ranks first with 72.27% score (avg@3), $6.18 per task, 1309 seconds per task.", "On MacroscopeBench, contributed by Macroscope, GPT-6 Astra · max (OpenAI) ranks first with 77.96% score (avg@3), $7.87 per task, 228.9 seconds per task.", "On ORCA Benchmark, contributed by Traversal, GPT-6 Astra (OpenAI) ranks first with 50.21% score (Hard RCA, avg@3), $4 per task, 1143.7 seconds per task.", "On Bedside Bench, contributed by Doximity, Doximity Ask V7 (Doximity) ranks first with 95.4% score (avg@3).", "On APEX-1: General Practitioner, contributed by Mercor, Claude Opus 5 (Anthropic) ranks first with 72.12% score (avg@4), $1.97 per task.", "On HealthBench Professional, GPT-6 Astra (OpenAI) ranks first with 64.43% score (avg@8), $0.0533 per task, 26.5 seconds per task.", "On Genspark Slides Benchmark, contributed by Genspark, Claude Fable 5.1 (Anthropic) ranks first with 83.59% score." ], "domains": [ { "name": "Security", "benchmark_ids": [ "defensive/dfbench", "offensive/pwnbench-v0-1", "offensive/cybergym" ] }, { "name": "Legal", "benchmark_ids": [ "legal/lab", "legal/lab-contracts", "legal/apex-agents-corporate-law", "legal/redline-bench" ] }, { "name": "Finance", "benchmark_ids": [ "finance/big-finance-benchmark" ] }, { "name": "Customer Support", "benchmark_ids": [ "customer-support/duetbench-diagnosis", "customer-support/tau3-banking", "customer-support/voice" ] }, { "name": "Software", "benchmark_ids": [ "code-generation/frontierswe-v2", "code-generation/apex-swe", "code-generation/deepswe-v1-1", "code-review/macroscope-code-review-benchmark", "site-reliability-engineering/traversal-orca-benchmark" ] }, { "name": "Healthcare", "benchmark_ids": [ "healthcare/bedside-bench", "healthcare/apex-1-general-practitioner", "healthcare/healthbench-professional" ] }, { "name": "Productivity", "benchmark_ids": [ "slide-generation/genspark-slides-benchmark" ] } ], "contributors": [ { "name": "Harvey", "coming_soon": false }, { "name": "Mercor", "coming_soon": false }, { "name": "Proximal", "coming_soon": false }, { "name": "Doximity", "coming_soon": false }, { "name": "Decagon", "coming_soon": false }, { "name": "Sierra", "coming_soon": false }, { "name": "Traversal", "coming_soon": false }, { "name": "depthfirst", "coming_soon": false }, { "name": "Novee", "coming_soon": false }, { "name": "Macroscope", "coming_soon": false }, { "name": "Genspark", "coming_soon": false }, { "name": "Rogo", "coming_soon": false }, { "name": "Heidi", "coming_soon": true }, { "name": "Chip Agents", "coming_soon": true } ], "benchmark_count": 20, "benchmarks": [ { "id": "defensive/dfbench", "name": "dfbench", "domain": "Security", "industry": "Defensive", "contributor": "depthfirst", "description": "A held-out evaluation of three jobs a security engineer actually does: finding vulnerabilities in a codebase, judging whether a reported finding is real, and working out what a diff changed. Detection and differential analysis run on separate task sets with their own ground truth, so the two are scored and costed independently rather than averaged into one number.", "task_count": null, "status": "published", "headline_metric": { "key": "detRecall", "label": "Score (detection recall)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://depthfirst.com", "source_title": "Open depthfirst", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-defensive%2Fdfbench", "methodology_summary": "depthfirst’s held-out evaluation of open-ended defensive-security work across vulnerability detection, finding validation, and differential analysis.", "board_note": "† Run with Claude Security.", "result_count": 12, "results": [ { "rank": 1, "ranked": true, "model": "Mythos 5†", "provider": "Anthropic", "metrics": { "score": { "value": 69, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": null, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 24.5, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 36.2, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 99.19, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": null, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Mythos 5†", "provider": "Anthropic", "detRecall": 69, "detCost": 99.19, "precision": 24.5, "f1": 36.2, "diffRecall": null, "diffCost": null, "cost": null } }, { "rank": 2, "ranked": true, "model": "GPT 5.6 Sol", "provider": "OpenAI · xhigh", "metrics": { "score": { "value": 65.7, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 27.02, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 75.3, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 18, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 28.3, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 43.37, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 10.67, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT 5.6 Sol", "provider": "OpenAI · xhigh", "detRecall": 65.7, "detCost": 43.37, "precision": 18, "f1": 28.3, "diffRecall": 75.3, "diffCost": 10.67, "cost": 27.02 } }, { "rank": 3, "ranked": true, "model": "dfs-large1", "provider": "depthfirst", "metrics": { "score": { "value": 62.2, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 4.055, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 75.6, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 18.3, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 28.3, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 6.77, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 1.34, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "dfs-large1", "provider": "depthfirst", "detRecall": 62.2, "detCost": 6.77, "precision": 18.3, "f1": 28.3, "diffRecall": 75.6, "diffCost": 1.34, "cost": 4.055 } }, { "rank": 4, "ranked": true, "model": "DeepSeek v4.1 Flash", "provider": "DeepSeek · high", "metrics": { "score": { "value": 57.3, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 1.2, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 70.1, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 36.4, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 44.5, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 1.69, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 0.71, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek v4.1 Flash", "provider": "DeepSeek · high", "detRecall": 57.3, "detCost": 1.69, "precision": 36.4, "f1": 44.5, "diffRecall": 70.1, "diffCost": 0.71, "cost": 1.2 } }, { "rank": 5, "ranked": true, "model": "Grok 4.5", "provider": "xAI · high", "metrics": { "score": { "value": 57.2, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 5.54, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 65.1, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 29.5, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 38.9, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 7.7, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 3.38, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.5", "provider": "xAI · high", "detRecall": 57.2, "detCost": 7.7, "precision": 29.5, "f1": 38.9, "diffRecall": 65.1, "diffCost": 3.38, "cost": 5.54 } }, { "rank": 6, "ranked": true, "model": "GPT 5.6 Luna", "provider": "OpenAI · xhigh", "metrics": { "score": { "value": 52.4, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 1.575, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 71, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 23.2, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 32.2, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 2.53, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 0.62, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT 5.6 Luna", "provider": "OpenAI · xhigh", "detRecall": 52.4, "detCost": 2.53, "precision": 23.2, "f1": 32.2, "diffRecall": 71, "diffCost": 0.62, "cost": 1.575 } }, { "rank": 7, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI · max", "metrics": { "score": { "value": 48, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 7.72, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 60.3, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 30.1, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 37, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 9.1, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 6.34, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI · max", "detRecall": 48, "detCost": 9.1, "precision": 30.1, "f1": 37, "diffRecall": 60.3, "diffCost": 6.34, "cost": 7.72 } }, { "rank": 8, "ranked": true, "model": "Opus 5", "provider": "Anthropic · med", "metrics": { "score": { "value": 47.8, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 13.685, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 71.3, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 40.5, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 43.8, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 21.89, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 5.48, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Opus 5", "provider": "Anthropic · med", "detRecall": 47.8, "detCost": 21.89, "precision": 40.5, "f1": 43.8, "diffRecall": 71.3, "diffCost": 5.48, "cost": 13.685 } }, { "rank": 9, "ranked": true, "model": "Muse Spark 1.3", "provider": "Meta · xhigh", "metrics": { "score": { "value": 47.4, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 6.68, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 72.8, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 34.1, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 39.7, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 10.24, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 3.12, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Muse Spark 1.3", "provider": "Meta · xhigh", "detRecall": 47.4, "detCost": 10.24, "precision": 34.1, "f1": 39.7, "diffRecall": 72.8, "diffCost": 3.12, "cost": 6.68 } }, { "rank": 10, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google · high", "metrics": { "score": { "value": 44.2, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 4.8, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 63.8, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 41.4, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 42.8, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 8.03, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 1.57, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google · high", "detRecall": 44.2, "detCost": 8.03, "precision": 41.4, "f1": 42.8, "diffRecall": 63.8, "diffCost": 1.57, "cost": 4.8 } }, { "rank": 11, "ranked": true, "model": "Qwen 3.8", "provider": "Alibaba · max", "metrics": { "score": { "value": 40.8, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 5.45, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 72.7, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 29.6, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 34.3, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 7.91, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 2.99, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Qwen 3.8", "provider": "Alibaba · max", "detRecall": 40.8, "detCost": 7.91, "precision": 29.6, "f1": 34.3, "diffRecall": 72.7, "diffCost": 2.99, "cost": 5.45 } }, { "rank": 12, "ranked": true, "model": "GLM 5.2", "provider": "Z.ai · xhigh", "metrics": { "score": { "value": 40.7, "unit": "percent", "label": "Score (detection recall)", "source_key": "detRecall" }, "cost_per_task": { "value": 5.88, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "diffRecall": { "value": 61.4, "unit": "percent", "label": "Differential-analysis recall", "source_key": "diffRecall" }, "precision": { "value": 28.6, "unit": "percent", "label": "Precision", "source_key": "precision" }, "f1": { "value": 33.6, "unit": "percent", "label": "F1", "source_key": "f1" }, "detCost": { "value": 9.72, "unit": "USD", "label": "Detection cost / task", "source_key": "detCost" }, "diffCost": { "value": 2.04, "unit": "USD", "label": "Differential-analysis cost / task", "source_key": "diffCost" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.2", "provider": "Z.ai · xhigh", "detRecall": 40.7, "detCost": 9.72, "precision": 28.6, "f1": 33.6, "diffRecall": 61.4, "diffCost": 2.04, "cost": 5.880000000000001 } } ] }, { "id": "offensive/pwnbench-v0-1", "name": "PWNBench v0.1", "domain": "Security", "industry": "Offensive", "contributor": "Novee", "description": "Greybox pentesting against realistic web applications: the agent probes a running target and files the vulnerabilities it finds. Reports are scored on precision and recall against a known ground truth, weighted by severity, alongside the API cost per task of the attempt. Every model is run at several attempt budgets so coverage and spend can be read together.", "task_count": null, "status": "published", "headline_metric": { "key": "f05", "label": "Score (F0.5)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://novee.security", "source_title": "Open Novee", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-offensive%2Fpwnbench-v0-1", "methodology_summary": "Novee’s end-to-end greybox penetration-testing benchmark for live web applications under a fixed, simplified harness.", "board_note": null, "result_count": 14, "results": [ { "rank": 1, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 63.3132, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 330.65, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 83.26, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 38.37, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 67.0087, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 80.14, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 48.59, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "baseModel": "Grok 4.6", "reasoning": "xhigh", "k": 3, "provider": "xAI", "precision": 83.26, "recall": 38.37, "f05": 63.3132, "mediumPrecision": 80.14, "mediumRecall": 48.59, "mediumF05": 67.0087, "cost": 330.65, "acc": 63.3132, "se": null, "lat": null } }, { "rank": 2, "ranked": true, "model": "Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 59.9613, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 1399.69, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 65.58, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 50.56, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 61.5009, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 68.85, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 49.24, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Opus 5", "baseModel": "Opus 5", "reasoning": "max", "k": 3, "provider": "Anthropic", "precision": 65.58, "recall": 50.56, "f05": 59.9613, "mediumPrecision": 68.85, "mediumRecall": 49.24, "mediumF05": 61.5009, "cost": 1399.69, "acc": 59.9613, "se": null, "lat": null } }, { "rank": 3, "ranked": true, "model": "Opus 4.8", "provider": "Anthropic", "metrics": { "score": { "value": 55.5876, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 564.34, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 74.55, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 28.99, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 61.8151, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 83.06, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 36.97, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Opus 4.8", "baseModel": "Opus 4.8", "reasoning": "max", "k": 3, "provider": "Anthropic", "precision": 74.55, "recall": 28.99, "f05": 55.5876, "mediumPrecision": 83.06, "mediumRecall": 36.97, "mediumF05": 61.8151, "cost": 564.34, "acc": 55.5876, "se": null, "lat": null } }, { "rank": 4, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 54.0096, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 571.05, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 74.93, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 31.21, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 52.8045, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 69.25, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 32.25, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "baseModel": "GPT-5.6 Sol", "reasoning": "max", "k": 3, "provider": "OpenAI", "precision": 74.93, "recall": 31.21, "f05": 54.0096, "mediumPrecision": 69.25, "mediumRecall": 32.25, "mediumF05": 52.8045, "cost": 571.05, "acc": 54.0096, "se": null, "lat": null } }, { "rank": 5, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 49.1864, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 15.25, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 52.14, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 40.1, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 51.9658, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 54.46, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 43.92, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "baseModel": "DeepSeek V4.1 Flash", "reasoning": "No Thinking", "k": 4, "provider": "DeepSeek", "precision": 52.14, "recall": 40.1, "f05": 49.1864, "mediumPrecision": 54.46, "mediumRecall": 43.92, "mediumF05": 51.9658, "cost": 15.25, "acc": 49.1864, "se": null, "lat": null } }, { "rank": 6, "ranked": true, "model": "Grok 4.5", "provider": "xAI", "metrics": { "score": { "value": 49.1222, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 100.07, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 60.34, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 30.52, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 50.8918, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 62.08, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 33.17, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.5", "baseModel": "Grok 4.5", "reasoning": "xhigh", "k": 3, "provider": "xAI", "precision": 60.34, "recall": 30.52, "f05": 49.1222, "mediumPrecision": 62.08, "mediumRecall": 33.17, "mediumF05": 50.8918, "cost": 100.07, "acc": 49.1222, "se": null, "lat": null } }, { "rank": 7, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 48.7517, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 173.93, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 60, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 27.86, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 55.814, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 71.7, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 29.59, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "baseModel": "GLM 5.3", "reasoning": "low", "k": 3, "provider": "Z.ai", "precision": 60, "recall": 27.86, "f05": 48.7517, "mediumPrecision": 71.7, "mediumRecall": 29.59, "mediumF05": 55.814, "cost": 173.93, "acc": 48.7517, "se": null, "lat": null } }, { "rank": 8, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 48.1912, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 208.66, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 50.59, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 41.99, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 52.1741, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 58.94, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 41.32, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "baseModel": "Kimi K3", "reasoning": "xhigh", "k": 3, "provider": "Moonshot AI", "precision": 50.59, "recall": 41.99, "f05": 48.1912, "mediumPrecision": 58.94, "mediumRecall": 41.32, "mediumF05": 52.1741, "cost": 208.66, "acc": 48.1912, "se": null, "lat": null } }, { "rank": 9, "ranked": true, "model": "Opus 4.6", "provider": "Anthropic", "metrics": { "score": { "value": 47.9068, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 311.07, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 61.77, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 27.86, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 57.2601, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 78.77, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 32.96, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Opus 4.6", "baseModel": "Opus 4.6", "reasoning": "max", "k": 3, "provider": "Anthropic", "precision": 61.77, "recall": 27.86, "f05": 47.9068, "mediumPrecision": 78.77, "mediumRecall": 32.96, "mediumF05": 57.2601, "cost": 311.07, "acc": 47.9068, "se": null, "lat": null } }, { "rank": 10, "ranked": true, "model": "DS-V4-Flash-0731", "provider": "DeepSeek", "metrics": { "score": { "value": 47.3797, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 56.46, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 63.52, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 27.73, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 48.4328, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 74.24, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 29.96, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 9, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DS-V4-Flash-0731", "baseModel": "DS-V4-Flash-0731", "reasoning": "No Thinking", "k": 9, "provider": "DeepSeek", "precision": 63.52, "recall": 27.73, "f05": 47.3797, "mediumPrecision": 74.24, "mediumRecall": 29.96, "mediumF05": 48.4328, "cost": 56.46, "acc": 47.3797, "se": null, "lat": null } }, { "rank": 11, "ranked": true, "model": "GLM 5.2", "provider": "Z.ai", "metrics": { "score": { "value": 44.2628, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 234.9, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 48.2, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 35.12, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 43.3908, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 51.18, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 30.08, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.2", "baseModel": "GLM 5.2", "reasoning": "No Thinking", "k": 3, "provider": "Z.ai", "precision": 48.2, "recall": 35.12, "f05": 44.2628, "mediumPrecision": 51.18, "mediumRecall": 30.08, "mediumF05": 43.3908, "cost": 234.9, "acc": 44.2628, "se": null, "lat": null } }, { "rank": 12, "ranked": true, "model": "GLM 5.3 Flash", "provider": "Z.ai", "metrics": { "score": { "value": 37.9708, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 36.84, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 52.98, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 17.8, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 40.3638, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 60.42, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 17.34, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3 Flash", "baseModel": "GLM 5.3 Flash", "reasoning": "low", "k": 4, "provider": "Z.ai", "precision": 52.98, "recall": 17.8, "f05": 37.9708, "mediumPrecision": 60.42, "mediumRecall": 17.34, "mediumF05": 40.3638, "cost": 36.84, "acc": 37.9708, "se": null, "lat": null } }, { "rank": 13, "ranked": true, "model": "Muse-Glimmer-30B", "provider": "Meta", "metrics": { "score": { "value": 25.8485, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 52.62, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 33.56, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 13.58, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 29.8485, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 43.7, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 14.08, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 9, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Muse-Glimmer-30B", "baseModel": "Muse-Glimmer-30B", "reasoning": "high", "k": 9, "provider": "Meta", "precision": 33.56, "recall": 13.58, "f05": 25.8485, "mediumPrecision": 43.7, "mediumRecall": 14.08, "mediumF05": 29.8485, "cost": 52.62, "acc": 25.8485, "se": null, "lat": null } }, { "rank": 14, "ranked": true, "model": "Nemotron 3 Ultra", "provider": "NVIDIA", "metrics": { "score": { "value": 18.7563, "unit": "percent", "label": "Score (F0.5)", "source_key": "f05" }, "cost_per_task": { "value": 330.12, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "precision": { "value": 22.55, "unit": "percent", "label": "Precision", "source_key": "precision" }, "recall": { "value": 9.18, "unit": "percent", "label": "Recall", "source_key": "recall" }, "mediumF05": { "value": 20.9019, "unit": "percent", "label": "F0.5 (medium and above)", "source_key": "mediumF05" }, "mediumPrecision": { "value": 24.25, "unit": "percent", "label": "Precision (medium and above)", "source_key": "mediumPrecision" }, "mediumRecall": { "value": 11.94, "unit": "percent", "label": "Recall (medium and above)", "source_key": "mediumRecall" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Nemotron 3 Ultra", "baseModel": "Nemotron 3 Ultra", "reasoning": "max", "k": 3, "provider": "NVIDIA", "precision": 22.55, "recall": 9.18, "f05": 18.7563, "mediumPrecision": 24.25, "mediumRecall": 11.94, "mediumF05": 20.9019, "cost": 330.12, "acc": 18.7563, "se": null, "lat": null } } ] }, { "id": "offensive/cybergym", "name": "CyberGym", "domain": "Security", "industry": "Offensive", "contributor": null, "description": "Level 1 of CyberGym asks a model to reproduce 1,507 real C/C++ vulnerabilities in their upstream projects: given the repository and the advisory, produce an input that triggers the bug. A task scores only when the crash actually reproduces, so there is no partial credit for a plausible-looking attempt.", "task_count": null, "status": "published", "headline_metric": { "key": "score", "label": "Score (pass@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": null, "source_title": null, "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-offensive%2Fcybergym", "methodology_summary": "Reproduction of real C/C++ software vulnerabilities by creating proof-of-concept inputs that distinguish vulnerable and fixed builds.", "board_note": "GPT-5.6 Sol and Claude Opus 5 rejected the CyberGym task requests; no tasks were scored.", "result_count": 6, "results": [ { "rank": 1, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 94.82, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": 6.53, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1353, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": "93.59–95.83%", "per_run_scores": null, "coverage": "3 × 1,507", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "score": 94.82, "ci": { "lo": 93.59, "hi": 95.83 }, "cost": 6.53, "lat": 1353, "runs": 3, "coverage": "3 × 1,507", "acc": 94.82, "se": null, "rankEligible": true } }, { "rank": 2, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 92.77, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": 5.69, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1283, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": "91.35–93.97%", "per_run_scores": null, "coverage": "3 × 1,507", "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "score": 92.77, "ci": { "lo": 91.35, "hi": 93.97 }, "cost": 5.69, "lat": 1283, "runs": 3, "coverage": "3 × 1,507", "acc": 92.77, "se": null, "rankEligible": true } }, { "rank": 3, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 92.1, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": 8.83, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1645, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": "90.63–93.36%", "per_run_scores": null, "coverage": "3 × 1,507", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "score": 92.1, "ci": { "lo": 90.63, "hi": 93.36 }, "cost": 8.83, "lat": 1645, "runs": 3, "coverage": "3 × 1,507", "acc": 92.1, "se": null, "rankEligible": true } }, { "rank": 4, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 90.18, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": 4.9, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1657, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": "88.57–91.58%", "per_run_scores": null, "coverage": "3 × 1,507", "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "score": 90.18, "ci": { "lo": 88.57, "hi": 91.58 }, "cost": 4.9, "lat": 1657, "runs": 3, "coverage": "3 × 1,507", "acc": 90.18, "se": null, "rankEligible": true } }, { "rank": null, "ranked": false, "model": "GPT-5.6 Sol*", "provider": "OpenAI · rejected tasks", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "0 scored", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol*", "provider": "OpenAI · rejected tasks", "score": null, "ci": null, "cost": null, "lat": null, "runs": 0, "coverage": "0 scored", "acc": null, "se": null, "rankEligible": false } }, { "rank": null, "ranked": false, "model": "Claude Opus 5*", "provider": "Anthropic · rejected tasks", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (pass@3)", "source_key": "score" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "0 scored", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5*", "provider": "Anthropic · rejected tasks", "score": null, "ci": null, "cost": null, "lat": null, "runs": 0, "coverage": "0 scored", "acc": null, "se": null, "rankEligible": false } } ] }, { "id": "legal/lab", "name": "LAB", "domain": "Legal", "industry": "Legal", "contributor": "Harvey", "description": "Harvey's Legal Agent Benchmark hands a model the work a first-year associate would get: a partner-style instruction, a client matter holding both relevant and peripheral files, and a deliverable someone has to review. Tasks span 24 transactional, advisory, regulatory, and litigation practice areas. Two independent judges grade expert-written rubric criteria; the headline AP score awards half credit when only one judge gives an all-pass result.", "task_count": 120, "status": "published", "headline_metric": { "key": "acc", "label": "Score (all-pass)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://www.harvey.ai", "source_title": "Open Harvey", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-legal%2Flab", "methodology_summary": "Harvey’s Legal Agent Benchmark evaluates long-horizon legal agents on reviewable work product across 24 law-firm practice areas. The Specialized Intelligence Index reports Fireworks runs on Harvey’s 120-task held-out set using Harvey’s dual-judge methodology and three runs per task.", "board_note": null, "result_count": 8, "results": [ { "rank": 1, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 22.08, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 4, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2160, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 15, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 88.07, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 22.08, "se": null, "cost": 4, "lat": 2160, "strictPass": 15, "criteriaPass": 88.07, "costText": "~$4.00* (tiered est)", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 2, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 14.72, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 4.29, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1560, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 9.72, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 93.62, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "acc": 14.72, "se": null, "cost": 4.29, "lat": 1560, "strictPass": 9.72, "criteriaPass": 93.62, "costText": "$4.29*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 3, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 13.06, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 4.86, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2340, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 9.72, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 91.87, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 13.06, "se": null, "cost": 4.86, "lat": 2340, "strictPass": 9.72, "criteriaPass": 91.87, "costText": "$4.86*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 4, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 7.36, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 17.88, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2280, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 6.11, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 68.08, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 7.36, "se": null, "cost": 17.88, "lat": 2280, "strictPass": 6.11, "criteriaPass": 68.08, "costText": "~$17.88*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 5, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 5.69, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 0.21, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1500, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 3.61, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 78.02, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "acc": 5.69, "se": null, "cost": 0.21, "lat": 1500, "strictPass": 3.61, "criteriaPass": 78.02, "costText": "~$0.21*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 6, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 5.14, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 28, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 600, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 2.5, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 90.45, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 5.14, "se": null, "cost": 28, "lat": 600, "strictPass": 2.5, "criteriaPass": 90.45, "costText": "~$28", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 7, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 3.61, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 0.93, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1380, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 1.94, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 86.49, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "acc": 3.61, "se": null, "cost": 0.93, "lat": 1380, "strictPass": 1.94, "criteriaPass": 86.49, "costText": "~$0.93*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 8, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 0.56, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 1.29, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 360, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 0, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 82.24, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 0.56, "se": null, "cost": 1.29, "lat": 360, "strictPass": 0, "criteriaPass": 82.24, "costText": "~$1.29*", "costRange": null, "url": "", "trainedWithFireworks": false } } ] }, { "id": "legal/lab-contracts", "name": "LAB: Contracts", "domain": "Legal", "industry": "Legal", "contributor": "Harvey", "description": "The contracts slice of Harvey's Legal Agent Benchmark, covering review, redlining, and issue-spotting on real transaction documents. Two independent judges grade expert-written rubric criteria; the headline AP score awards half credit when only one judge gives an all-pass result.", "task_count": 50, "status": "published", "headline_metric": { "key": "acc", "label": "Score (all-pass)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://www.harvey.ai", "source_title": "Open Harvey", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-legal%2Flab-contracts", "methodology_summary": "LAB: Contracts extends LAB to in-house contract drafting, review, and negotiation. The Specialized Intelligence Index reports Fireworks runs on Harvey’s 50-task held-out set using Harvey’s dual-judge methodology and three attempts per task.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 10, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 2.19, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2160, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 4.67, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 81.47, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 10, "se": null, "cost": 2.19, "lat": 2160, "strictPass": 4.67, "criteriaPass": 81.47, "costText": "$2.19*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 2, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 8.33, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 5.54, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2100, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 4, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 89.75, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 8.33, "se": null, "cost": 5.54, "lat": 2100, "strictPass": 4, "criteriaPass": 89.75, "costText": "$5.54*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 3, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 6.33, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 0.91, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 360, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 3.33, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 74.42, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 6.33, "se": null, "cost": 0.91, "lat": 360, "strictPass": 3.33, "criteriaPass": 74.42, "costText": "~$0.91*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 4, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 6, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 4.31, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1560, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 1.33, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 89.76, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "acc": 6, "se": null, "cost": 4.31, "lat": 1560, "strictPass": 1.33, "criteriaPass": 89.76, "costText": "$4.31*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 5, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 5.33, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 28, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 480, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 1.33, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 89.48, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 5.33, "se": null, "cost": 28, "lat": 480, "strictPass": 1.33, "criteriaPass": 89.48, "costText": "~$28", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 6, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 2, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 1.16, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 240, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 1.33, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 85.13, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 2, "se": null, "cost": 1.16, "lat": 240, "strictPass": 1.33, "criteriaPass": 85.13, "costText": "~$1.16*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 7, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 2, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 8.08, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1200, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 0, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 35.02, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 2, "se": null, "cost": 8.08, "lat": 1200, "strictPass": 0, "criteriaPass": 35.02, "costText": "~$8.08*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 1, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 0.93, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 660, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 0, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 81.13, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "acc": 1, "se": null, "cost": 0.93, "lat": 660, "strictPass": 0, "criteriaPass": 81.13, "costText": "$0.93*", "costRange": null, "url": "", "trainedWithFireworks": false } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 0.33, "unit": "percent", "label": "Score (all-pass)", "source_key": "acc" }, "cost_per_task": { "value": 0.29, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1080, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "strictPass": { "value": 0, "unit": "percent", "label": "Score (Strict: both judges all-pass)", "source_key": "strictPass" }, "criteriaPass": { "value": 23.97, "unit": "percent", "label": "Score (Crit: criteria pass rate)", "source_key": "criteriaPass" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "acc": 0.33, "se": null, "cost": 0.29, "lat": 1080, "strictPass": 0, "criteriaPass": 23.97, "costText": "~$0.29*", "costRange": null, "url": "", "trainedWithFireworks": false } } ] }, { "id": "legal/apex-agents-corporate-law", "name": "APEX Agents: Corporate Law", "domain": "Legal", "industry": "Legal", "contributor": "Mercor", "description": "Multi-step corporate-law workflows that chain tool use, retrieval, and document drafting into a single assignment. Practicing corporate attorneys grade the output against the work product a firm would actually accept.", "task_count": 68, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@4)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://www.mercor.com", "source_title": "Open Mercor", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-legal%2Fapex-agents-corporate-law", "methodology_summary": "APEX-Agents 1.1 evaluates long-horizon corporate-law work across realistic files and productivity tools. The Specialized Intelligence Index publishes the supplied four-attempt mean rubric scores on a common 68-task slice with reported 95% confidence intervals.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 87.67, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 4.422, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 73.16, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 9.74, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 1.1054, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 358442, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "81.69–93.65%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "servedBy": "OpenAI", "modelIdentifier": "responses/vega-alpha", "reasoning": "max", "acc": 87.67, "ci": { "lo": 81.69, "hi": 93.65, "halfWidth": 5.98 }, "pass1": 73.16, "pass1CiHalfWidth": 9.74, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 4.422, "costPerAttempt": 1.1054, "avgTokens": 358442, "k": 4, "priceBasis": "openai/gpt-6-astra", "batchId": "batch_bd4226183a0942c886a41674945d63d9" } }, { "rank": 2, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 83.8, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 5.772, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 69.49, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 9.93, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 1.4483, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 271, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 1330916, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "77.10–90.50%", "per_run_scores": null, "coverage": "271 / 272 (99.63%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "servedBy": "Anthropic", "modelIdentifier": "claude-opus-5", "reasoning": "max", "acc": 83.8, "ci": { "lo": 77.1, "hi": 90.5, "halfWidth": 6.7 }, "pass1": 69.49, "pass1CiHalfWidth": 9.93, "attemptsScored": 271, "attemptsExpected": 272, "coverage": "271 / 272 (99.63%)", "cost": 5.772, "costPerAttempt": 1.4483, "avgTokens": 1330916, "k": 4, "priceBasis": "anthropic/claude-opus-5", "batchId": "batch_52319d6c9dc94f10b2b4ce8c7c304591" } }, { "rank": 3, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 82.71, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 2.386, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 63.81, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 10.63, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.5965, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 268, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2484096, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "76.09–89.33%", "per_run_scores": null, "coverage": "268 / 272 (98.53%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "servedBy": "Google Vertex", "modelIdentifier": "gemini-3.8-flash", "reasoning": "high", "acc": 82.71, "ci": { "lo": 76.09, "hi": 89.33, "halfWidth": 6.62 }, "pass1": 63.81, "pass1CiHalfWidth": 10.63, "attemptsScored": 268, "attemptsExpected": 272, "coverage": "268 / 272 (98.53%)", "cost": 2.386, "costPerAttempt": 0.5965, "avgTokens": 2484096, "k": 4, "priceBasis": "vertex_ai/gemini-3.8-flash", "batchId": "batch_c803679148594bea8da6bf12775c8cd1" } }, { "rank": 4, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 82.21, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 2.098, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 65.07, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 9.93, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.5245, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 605475, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "75.72–88.70%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "servedBy": "xAI", "modelIdentifier": "grok-4.6", "reasoning": "xhigh", "acc": 82.21, "ci": { "lo": 75.72, "hi": 88.7, "halfWidth": 6.49 }, "pass1": 65.07, "pass1CiHalfWidth": 9.93, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 2.098, "costPerAttempt": 0.5245, "avgTokens": 605475, "k": 4, "priceBasis": "xai/grok-4.6", "batchId": "batch_54925f40249e48b8aad9a1c99bfab245" } }, { "rank": 5, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 81.41, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 2.492, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 67.65, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 9.56, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.623, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 784585, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "74.65–88.17%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/glm-5p3", "reasoning": "max", "acc": 81.41, "ci": { "lo": 74.65, "hi": 88.17, "halfWidth": 6.76 }, "pass1": 67.65, "pass1CiHalfWidth": 9.56, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 2.492, "costPerAttempt": 0.623, "avgTokens": 784585, "k": 4, "priceBasis": "fireworks_ai/.../glm-5p3", "batchId": "batch_7138d11ea41e4418a7f25d8bc79929b2" } }, { "rank": 6, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 80.61, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 11.996, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 62.5, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 10.11, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 2.999, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2836700, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "73.85–87.37%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "servedBy": "OpenAI", "modelIdentifier": "responses/gpt-5.6-sol", "reasoning": "max", "acc": 80.61, "ci": { "lo": 73.85, "hi": 87.37, "halfWidth": 6.76 }, "pass1": 62.5, "pass1CiHalfWidth": 10.11, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 11.996, "costPerAttempt": 2.999, "avgTokens": 2836700, "k": 4, "priceBasis": "openai/gpt-5.6-sol", "batchId": "batch_cbfc0925a1be4353846ea2d4c5c259b9" } }, { "rank": 7, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 79.28, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 3.599, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 61.76, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 9.93, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.8997, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 1433792, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "72.50–86.06%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4-pro-0813", "reasoning": "max", "acc": 79.28, "ci": { "lo": 72.5, "hi": 86.06, "halfWidth": 6.78 }, "pass1": 61.76, "pass1CiHalfWidth": 9.93, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 3.599, "costPerAttempt": 0.8997, "avgTokens": 1433792, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4-pro-0813", "batchId": "batch_6a7df85ba60f4316b9e2844504ac7a54" } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 78.37, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.56, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 61.76, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 10.11, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.1399, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 3556765, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "70.93–85.81%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4p1-flash", "reasoning": "max", "acc": 78.37, "ci": { "lo": 70.93, "hi": 85.81, "halfWidth": 7.44 }, "pass1": 61.76, "pass1CiHalfWidth": 10.11, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 0.56, "costPerAttempt": 0.1399, "avgTokens": 3556765, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4p1-flash", "batchId": "batch_bdae5926780f47efbadaf959cfae2769" } }, { "rank": 9, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 76.84, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 3.147, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 62.13, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 10.11, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.7866, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 272, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 272, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 562174, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "69.50–84.18%", "per_run_scores": null, "coverage": "272 / 272 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/kimi-k3", "reasoning": "max", "acc": 76.84, "ci": { "lo": 69.5, "hi": 84.18, "halfWidth": 7.34 }, "pass1": 62.13, "pass1CiHalfWidth": 10.11, "attemptsScored": 272, "attemptsExpected": 272, "coverage": "272 / 272 (100%)", "cost": 3.147, "costPerAttempt": 0.7866, "avgTokens": 562174, "k": 4, "priceBasis": "fireworks_ai/.../kimi-k3", "batchId": "batch_88e223cce4514095a89822cd7e4676bd" } } ] }, { "id": "legal/redline-bench", "name": "Redline Bench", "domain": "Legal", "industry": "Legal", "contributor": "Crosby", "description": "RedlineBench seats a terminal agent as in-house counsel and asks it to negotiate a contract across four sequential turns, editing the document in place with tracked changes and comments. The counterparty's instructions and the grading rubric stay hidden from the agent throughout. Scoring is criterion-by-criterion across twelve equally weighted scenario-and-turn cells.", "task_count": 140, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": "https://www.crosby.ai", "source_title": "Open Crosby", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-legal%2Fredline-bench", "methodology_summary": "RedlineBench measures realistic, multi-turn contract redlining by a terminal agent acting as in-house counsel. The Specialized Intelligence Index holds the public task set, harness, task budget, verification, and aggregation fixed across every model it runs.", "board_note": null, "result_count": 8, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra (max)", "provider": "OpenAI", "metrics": { "score": { "value": 62.8421, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 9.2568, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2286.42, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 64.32, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": 147541.6667, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 3962725, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 3815183.3333, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 79324, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "60.97–64.71%", "per_run_scores": [ 63.2884, 61.973, 63.2649 ], "coverage": null, "note": "Max reasoning on the official OpenAI Responses API. Cached and uncached input are cost-derived from the recorded prompt total because the measured split was unavailable. 48 benchmark timeouts and two validity-gate zeros retained.", "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "displayModel": "GPT-6 Astra (max)", "provider": "OpenAI", "runs": [ 63.2884, 61.973, 63.2649 ], "cost": 9.2568, "lat": 2286.42, "tools": 64.32, "inputTokens": 3962725, "cachedTokens": 3815183.3333333335, "uncachedInput": 147541.6666666665, "cacheEstimated": true, "recordedOutput": 79324, "caveat": "Max reasoning on the official OpenAI Responses API. Cached and uncached input are cost-derived from the recorded prompt total because the measured split was unavailable. 48 benchmark timeouts and two validity-gate zeros retained.", "acc": 62.842099999999995, "runScores": "63.2884 · 61.9730 · 63.2649", "ci": { "lo": 60.972154434752596, "hi": 64.7120455652474, "halfWidth": 1.8699455652473995, "runs": 3 } } }, { "rank": 2, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 54.715, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 4.6886, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 3163.84, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 94.13, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 5222638, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 4539903, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 175537, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "49.80–59.63%", "per_run_scores": [ 53.9839, 56.9564, 53.2048 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "runs": [ 53.9839, 56.9564, 53.2048 ], "cost": 4.6886421190476195, "lat": 3163.84, "tools": 94.13, "inputTokens": 5222638, "cachedTokens": 4539903, "recordedOutput": 175537, "acc": 54.71503333333334, "runScores": "53.9839 · 56.9564 · 53.2048", "ci": { "lo": 49.796978544553745, "hi": 59.63308812211293, "halfWidth": 4.918054788779594, "runs": 3 } } }, { "rank": 3, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 50.6682, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 9.1063, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2835.26, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 103.56, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 7409399, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 7180707, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 163475, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "49.64–51.70%", "per_run_scores": [ 51.107, 50.6139, 50.2836 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "runs": [ 51.107, 50.6139, 50.2836 ], "cost": 9.106347220833333, "lat": 2835.26, "tools": 103.56, "inputTokens": 7409399, "cachedTokens": 7180707, "recordedOutput": 163475, "acc": 50.66816666666667, "runScores": "51.1070 · 50.6139 · 50.2836", "ci": { "lo": 49.638805391383706, "hi": 51.69752794194964, "halfWidth": 1.0293612752829635, "runs": 3 } } }, { "rank": 4, "ranked": true, "model": "GPT-5.6 Sol (medium)", "provider": "OpenAI", "metrics": { "score": { "value": 50.2755, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 1.1412, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 467.66, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 55.47, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": 73390.4949, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 1023497.902, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 950107.4071, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 18051.6857, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "47.47–53.08%", "per_run_scores": [ 51.5122, 50.0131, 49.3011 ], "coverage": null, "note": "Medium reasoning configuration; seven accepted replacements. Cached and uncached input are cost-derived estimates because the measured split was unavailable.", "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "displayModel": "GPT-5.6 Sol (medium)", "provider": "OpenAI", "runs": [ 51.5122, 50.0131, 49.3011 ], "cost": 1.141180289047619, "lat": 467.66, "tools": 55.47, "inputTokens": 1023497.902, "cachedTokens": 950107.407142857, "uncachedInput": 73390.49485714303, "cacheEstimated": true, "recordedOutput": 18051.68571, "caveat": "Medium reasoning configuration; seven accepted replacements. Cached and uncached input are cost-derived estimates because the measured split was unavailable.", "acc": 50.27546666666667, "runScores": "51.5122 · 50.0131 · 49.3011", "ci": { "lo": 47.471725730852434, "hi": 53.0792076024809, "halfWidth": 2.8037409358142353, "runs": 3 } } }, { "rank": 5, "ranked": true, "model": "GLM-5.3", "provider": "Z.ai", "metrics": { "score": { "value": 49.5649, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 2.7216, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1903.73, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 117.44, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 7059329, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 6677617, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 102513, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "46.22–52.91%", "per_run_scores": [ 48.2788, 50.9675, 49.4485 ], "coverage": null, "note": "Run 3 includes one accepted replacement; selected-task counters are used.", "fireworks_model_url": null, "raw": { "model": "GLM-5.3", "provider": "Z.ai", "runs": [ 48.2788, 50.9675, 49.4485 ], "cost": 2.721635238095238, "lat": 1903.73, "tools": 117.44, "inputTokens": 7059329, "cachedTokens": 6677617, "recordedOutput": 102513, "caveat": "Run 3 includes one accepted replacement; selected-task counters are used.", "acc": 49.56493333333333, "runScores": "48.2788 · 50.9675 · 49.4485", "ci": { "lo": 46.21600200400712, "hi": 52.913864662659535, "halfWidth": 3.348931329326208, "runs": 3 } } }, { "rank": 6, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 49.0439, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 3.9484, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1981.591, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 86.4381, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 5446045.4, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 5149960.755, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 101008.9976, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "48.05–50.04%", "per_run_scores": [ 48.888, 48.7438, 49.4998 ], "coverage": null, "note": "Run 3 replaced the prior retained run with a complete LiteLLM 1.99.0 run.", "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "runs": [ 48.888, 48.7438, 49.4998 ], "cost": 3.948378055, "lat": 1981.5914, "tools": 86.43809524, "inputTokens": 5446045.4, "cachedTokens": 5149960.755, "recordedOutput": 101008.9976, "caveat": "Run 3 replaced the prior retained run with a complete LiteLLM 1.99.0 run.", "acc": 49.043866666666666, "runScores": "48.8880 · 48.7438 · 49.4998", "ci": { "lo": 48.046786848173554, "hi": 50.04094648515978, "halfWidth": 0.9970798184931158, "runs": 3 } } }, { "rank": 7, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 46.9099, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.632, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 858.52, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 121.35, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 5392633, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 5250233, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 53791, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "42.25–51.57%", "per_run_scores": [ 45.2466, 48.9432, 46.54 ], "coverage": null, "note": "Run 1 retains 26 scored trajectories with rate-limit evidence.", "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "runs": [ 45.2466, 48.9432, 46.54 ], "cost": 0.6319928112666666, "lat": 858.52, "tools": 121.35, "inputTokens": 5392633, "cachedTokens": 5250233, "recordedOutput": 53791, "caveat": "Run 1 retains 26 scored trajectories with rate-limit evidence.", "acc": 46.90993333333333, "runScores": "45.2466 · 48.9432 · 46.5400", "ci": { "lo": 42.2500386633061, "hi": 51.569828003360556, "halfWidth": 4.659894670027225, "runs": 3 } } }, { "rank": 8, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 44.4785, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 1.8293, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 720.56, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "tools": { "value": 95.7, "unit": "count", "label": "Tool calls / task", "source_key": "tools" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 13295929, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 12553216, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 88215, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "41.20–47.76%", "per_run_scores": [ 45.2581, 45.222, 42.9553 ], "coverage": null, "note": "Run 2 includes one accepted replacement and one scored transient rate-limit task.", "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "runs": [ 45.2581, 45.222, 42.9553 ], "cost": 1.829332380952381, "lat": 720.56, "tools": 95.7, "inputTokens": 13295929, "cachedTokens": 12553216, "recordedOutput": 88215, "caveat": "Run 2 includes one accepted replacement and one scored transient rate-limit task.", "acc": 44.47846666666666, "runScores": "45.2581 · 45.2220 · 42.9553", "ci": { "lo": 41.201331296280266, "hi": 47.75560203705306, "halfWidth": 3.277135370386396, "runs": 3 } } } ] }, { "id": "finance/big-finance-benchmark", "name": "Big Finance Benchmark", "domain": "Finance", "industry": "Finance", "contributor": "Rogo", "description": "Rogo's Big Finance Benchmark evaluates financial-research agents on private, workflow-grounded tasks. Models use Rogo's official agent loop and grading methodology, and the headline measures final-answer accuracy across independent full runs.", "task_count": 139, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://github.com/Rogo-Technologies/big-finance-benchmark", "source_title": "Open the Big Finance Benchmark repository", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-finance%2Fbig-finance-benchmark", "methodology_summary": "Rogo's Big Finance Benchmark evaluates financial-research agents on a private 139-task holdout set. Fireworks runs Rogo's official Big Finance agent loop and publishes aggregates produced with one consistent Kimi K3 judge configuration.", "board_note": null, "result_count": 8, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 54.6, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.303, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 68.3, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 57.2, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 49.3, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 71.6, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 140300, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 119700, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 1200, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 300, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "fullRuns": 3, "fullRunsText": "3", "acc": 54.6, "rubricPct": 68.3, "prompt": 140300, "cached": 119700, "completion": 1200, "reasoningTokens": 300, "cost": 0.303, "costPerRun": 71.6, "passAt3": 57.2, "passAll3": 49.3 } }, { "rank": 2, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 49.2, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.501, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 66.3, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 53.2, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 43.2, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 259.54, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 342000, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 8100, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 5700, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "fullRuns": 3, "fullRunsText": "3", "acc": 49.2, "rubricPct": 66.3, "prompt": 342000, "cached": 0, "cachedMark": "†", "completion": 8100, "reasoningTokens": 5700, "cost": 0.501, "costPerRun": 259.54, "passAt3": 53.2, "passAll3": 43.2 } }, { "rank": 3, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 45.8, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.286, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 65.9, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 52.5, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 38.1, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 70.31, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 439500, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 400700, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 56400, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 53000, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "fullRuns": 3, "fullRunsText": "3", "acc": 45.8, "rubricPct": 65.9, "prompt": 439500, "cached": 400700, "completion": 56400, "reasoningTokens": 53000, "cost": 0.286, "costPerRun": 70.31, "passAt3": 52.5, "passAll3": 38.1 } }, { "rank": 4, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 44.2, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.226, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 60.8, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 53.6, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 34.1, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 44.36, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 176800, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 152200, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2500, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 1300, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "fullRuns": 3, "fullRunsText": "3", "acc": 44.2, "rubricPct": 60.8, "prompt": 176800, "cached": 152200, "completion": 2500, "reasoningTokens": 1300, "cost": 0.226, "costPerRun": 44.36, "passAt3": 53.6, "passAll3": 34.1 } }, { "rank": 5, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 41.3, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.22, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 62.3, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 51.4, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 31.9, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 48.27, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 265400, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 236300, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 6700, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 4100, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "fullRuns": 3, "fullRunsText": "3", "acc": 41.3, "rubricPct": 62.3, "prompt": 265400, "cached": 236300, "completion": 6700, "reasoningTokens": 4100, "cost": 0.22, "costPerRun": 48.27, "passAt3": 51.4, "passAll3": 31.9 } }, { "rank": 6, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 40.6, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.107, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 60.4, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 48.6, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 32.6, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 20.67, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 654900, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 610200, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 27500, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 24400, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "fullRuns": 3, "fullRunsText": "3", "acc": 40.6, "rubricPct": 60.4, "prompt": 654900, "cached": 610200, "completion": 27500, "reasoningTokens": 24400, "cost": 0.107, "costPerRun": 20.67, "passAt3": 48.6, "passAll3": 32.6 } }, { "rank": 7, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 39.3, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.141, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 58.1, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 47.5, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 31.7, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 32.39, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 3, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 444500, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 407900, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 18600, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 16200, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "fullRuns": 3, "fullRunsText": "3", "acc": 39.3, "rubricPct": 58.1, "prompt": 444500, "cached": 407900, "completion": 18600, "reasoningTokens": 16200, "cost": 0.141, "costPerRun": 32.39, "passAt3": 47.5, "passAll3": 31.7 } }, { "rank": 8, "ranked": true, "model": "GLM 5.3 Flash", "provider": "Z.ai", "metrics": { "score": { "value": 37.2, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.085, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "rubricPct": { "value": 58.9, "unit": "percent", "label": "Score (rubric avg@3)", "source_key": "rubricPct" }, "passAt3": { "value": 46.8, "unit": "percent", "label": "Score (pass@3)", "source_key": "passAt3" }, "passAll3": { "value": 27.3, "unit": "percent", "label": "Score (pass^3)", "source_key": "passAll3" }, "costPerRun": { "value": 14.66, "unit": "USD", "label": "Cost (median USD / run)", "source_key": "costPerRun" }, "fullRuns": { "value": 4, "unit": "count", "label": "Runs (full)", "source_key": "fullRuns" }, "prompt": { "value": 261100, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 219400, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 15700, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" }, "reasoningTokens": { "value": 13100, "unit": "tokens", "label": "Reasoning tokens / task", "source_key": "reasoningTokens" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3 Flash", "provider": "Z.ai", "fullRuns": 4, "fullRunsText": "4*", "acc": 37.2, "rubricPct": 58.9, "prompt": 261100, "cached": 219400, "completion": 15700, "reasoningTokens": 13100, "cost": 0.085, "costPerRun": 14.66, "passAt3": 46.8, "passAll3": 27.3 } } ] }, { "id": "customer-support/duetbench-diagnosis", "name": "DuetBench–Diagnosis", "domain": "Customer Support", "industry": "Customer Support", "contributor": "Decagon", "description": "Real Duet diagnosis conversations replayed from the first human turn. A blinded judge compares model responses head to head on task outcome, investigation, tool use, and communication; the four relative ratings are averaged into Diagnosis Elo.", "task_count": null, "status": "published", "headline_metric": { "key": "elo", "label": "Score (Diagnosis Elo)", "unit": "Elo points", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://decagon.ai/blog/duetbench", "source_title": "Open Decagon’s public DuetBench overview", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-customer-support%2Fduetbench-diagnosis", "methodology_summary": "Decagon’s diagnosis arena replays real Duet customer-support investigations and rates model responses head to head across outcome, investigation, tool use, and communication.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 3.8629, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 326.3, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 1034.9, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1039.7, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 1041.7, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1025.6, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 1032.4, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 82, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 508, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 100, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 563079, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 1165193, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 18597, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "994.50–1071.80%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "elo": 1034.9, "ci": { "lo": 994.5, "hi": 1071.8 }, "taskOutcome": 1039.7, "investigation": 1041.7, "toolUse": 1025.6, "communication": 1032.4, "battles": 508, "distinctCases": 100, "completionRate": 82, "lat": 326.3, "stalenessDays": 0, "outputTokens": 18597, "uncachedInput": 563079, "cachedInput": 1165193, "cost": 3.8629165, "acc": 1034.9 } }, { "rank": 2, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 0.8997, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 83.2, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 1031.5, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1035.5, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 1029.3, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1026.2, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 1034.8, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 98, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 240, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 50, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 477051, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 731938, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 9437, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "973.50–1078.60%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "elo": 1031.5, "ci": { "lo": 973.5, "hi": 1078.6 }, "taskOutcome": 1035.5, "investigation": 1029.3, "toolUse": 1026.2, "communication": 1034.8, "battles": 240, "distinctCases": 50, "completionRate": 98, "lat": 83.2, "stalenessDays": 0, "outputTokens": 9437, "uncachedInput": 477051, "cachedInput": 731938, "cost": 0.89969808, "acc": 1031.5 } }, { "rank": 3, "ranked": true, "model": "Claude 5 Sonnet", "provider": "Anthropic", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 0.9968, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 239.3, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 1022.1, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1019.2, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 1022.3, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1017.6, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 1029.6, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 95, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 425, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 100, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 347636, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 734031, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 15471, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "989.00–1056.10%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude 5 Sonnet", "provider": "Anthropic", "elo": 1022.1, "ci": { "lo": 989, "hi": 1056.1 }, "taskOutcome": 1019.2, "investigation": 1022.3, "toolUse": 1017.6, "communication": 1029.6, "battles": 425, "distinctCases": 100, "completionRate": 95, "lat": 239.3, "stalenessDays": 0, "outputTokens": 15471, "uncachedInput": 347636, "cachedInput": 734031, "cost": 0.9967881999999999, "acc": 1022.1 } }, { "rank": 4, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 1.2164, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 143.2, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 1011, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1006.3, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 1018.7, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1019.6, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 999.5, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 96, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 324, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 90, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 280570, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 643584, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 12106, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "974.90–1046.40%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "elo": 1011, "ci": { "lo": 974.9, "hi": 1046.4 }, "taskOutcome": 1006.3, "investigation": 1018.7, "toolUse": 1019.6, "communication": 999.5, "battles": 324, "distinctCases": 90, "completionRate": 96, "lat": 143.2, "stalenessDays": 0, "outputTokens": 12106, "uncachedInput": 280570, "cachedInput": 643584, "cost": 1.2163751999999999, "acc": 1011 } }, { "rank": 5, "ranked": true, "model": "GLM 5.2", "provider": "Z.ai", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 0.7641, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 189.3, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 1009.3, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1008.6, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 1020.6, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1007.2, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 1000.9, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 96, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 314, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 79, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 400548, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 824466, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 19989, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "968.90–1045.00%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.2", "provider": "Z.ai", "elo": 1009.3, "ci": { "lo": 968.9, "hi": 1045 }, "taskOutcome": 1008.6, "investigation": 1020.6, "toolUse": 1007.2, "communication": 1000.9, "battles": 314, "distinctCases": 79, "completionRate": 96, "lat": 189.3, "stalenessDays": 0, "outputTokens": 19989, "uncachedInput": 400548, "cachedInput": 824466, "cost": 0.7641440399999999, "acc": 1009.3 } }, { "rank": 6, "ranked": true, "model": "Claude Opus 4.6", "provider": "Anthropic", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 3.2718, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 315.2, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 997.9, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 1003.5, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 979.7, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 1015.4, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 992.9, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 97, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 120, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 30, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 517532, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 717019, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 13027, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "935.00–1057.30%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 4.6", "provider": "Anthropic", "elo": 997.9, "ci": { "lo": 935, "hi": 1057.3 }, "taskOutcome": 1003.5, "investigation": 979.7, "toolUse": 1015.4, "communication": 992.9, "battles": 120, "distinctCases": 30, "completionRate": 97, "lat": 315.2, "stalenessDays": 0, "outputTokens": 13027, "uncachedInput": 517532, "cachedInput": 717019, "cost": 3.2718445, "acc": 997.9 } }, { "rank": 7, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 2.1212, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 118.1, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 990.3, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 989.3, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 988.9, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 992, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 991, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 87, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 445, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 100, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 0, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 446818, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 465313, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 7392, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "962.60–1017.20%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "elo": 990.3, "ci": { "lo": 962.6, "hi": 1017.2 }, "taskOutcome": 989.3, "investigation": 988.9, "toolUse": 992, "communication": 991, "battles": 445, "distinctCases": 100, "completionRate": 87, "lat": 118.1, "stalenessDays": 0, "outputTokens": 7392, "uncachedInput": 446818, "cachedInput": 465313, "cost": 2.1212372000000004, "acc": 990.3 } }, { "rank": 8, "ranked": true, "model": "GPT-5.6 Terra", "provider": "OpenAI", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 0.5489, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 65.2, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 951.9, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 946.4, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 943.2, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 954.9, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 963.1, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 95, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 175, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 50, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 16, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 214734, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 295627, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 5024, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "907.80–995.10%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Terra", "provider": "OpenAI", "elo": 951.9, "ci": { "lo": 907.8, "hi": 995.1 }, "taskOutcome": 946.4, "investigation": 943.2, "toolUse": 954.9, "communication": 963.1, "battles": 175, "distinctCases": 50, "completionRate": 95, "lat": 65.2, "stalenessDays": 16, "outputTokens": 5024, "uncachedInput": 214734, "cachedInput": 295627, "cost": 0.5488814000000001, "acc": 951.9 } }, { "rank": 9, "ranked": true, "model": "GPT-5.6 Luna", "provider": "OpenAI", "metrics": { "score": { "value": null, "unit": "percent", "label": "Score (Diagnosis Elo)", "source_key": null }, "cost_per_task": { "value": 0.1019, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 85.9, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "elo": { "value": 951.1, "unit": "Elo points", "label": "Score (Diagnosis Elo)", "source_key": "elo" }, "taskOutcome": { "value": 951.4, "unit": "Elo points", "label": "Task Outcome Elo", "source_key": "taskOutcome" }, "investigation": { "value": 955.4, "unit": "Elo points", "label": "Investigation Elo", "source_key": "investigation" }, "toolUse": { "value": 941.4, "unit": "Elo points", "label": "Tool Use Elo", "source_key": "toolUse" }, "communication": { "value": 955.9, "unit": "Elo points", "label": "Communication Elo", "source_key": "communication" }, "completionRate": { "value": 79, "unit": "percent", "label": "Completion rate", "source_key": "completionRate" }, "battles": { "value": 195, "unit": "count", "label": "Pairwise battles", "source_key": "battles" }, "distinctCases": { "value": 59, "unit": "count", "label": "Distinct cases", "source_key": "distinctCases" }, "stalenessDays": { "value": 16, "unit": "days", "label": "Staleness", "source_key": "stalenessDays" }, "uncachedInput": { "value": 420012, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedInput": { "value": 425882, "unit": "tokens", "label": "Cached input tokens / task", "source_key": "cachedInput" }, "outputTokens": { "value": 7820, "unit": "tokens", "label": "Output tokens / task", "source_key": "outputTokens" } }, "confidence_interval_95": "916.40–990.20%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Luna", "provider": "OpenAI", "elo": 951.1, "ci": { "lo": 916.4, "hi": 990.2 }, "taskOutcome": 951.4, "investigation": 955.4, "toolUse": 941.4, "communication": 955.9, "battles": 195, "distinctCases": 59, "completionRate": 79, "lat": 85.9, "stalenessDays": 16, "outputTokens": 7820, "uncachedInput": 420012, "cachedInput": 425882, "cost": 0.10190404, "acc": 951.1 } } ] }, { "id": "customer-support/tau3-banking", "name": "τ³-Banking", "domain": "Customer Support", "industry": "Customer Support", "contributor": "Sierra", "description": "Policy-bound banking agents handle realistic customer requests with database tools and a simulated user. The board covers the 97-task banking_knowledge subset, with four official seeded trials per task and binary rewards for correct final state and required natural-language assertions.", "task_count": 97, "status": "published", "headline_metric": { "key": "acc", "label": "Score (mean trial reward)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": "https://github.com/sierra-research/tau2-bench", "source_title": "Open the Tau3-Bench repository", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-customer-support%2Ftau3-banking", "methodology_summary": "Tau3-Bench banking evaluates policy-bound customer-service agents on realistic banking requests that require conversation, tool use, and a correct final database state. The Specialized Intelligence Index runs and reports the 97-task banking_knowledge subset, not the official 375-task Tau3-Bench aggregate.", "board_note": null, "result_count": 11, "results": [ { "rank": 1, "ranked": true, "model": "GLM-5.3", "provider": "Z.ai", "metrics": { "score": { "value": 51.6323, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.4852, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 281.6, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 108334, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1056419, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 13382, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "47.72–55.54%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM-5.3", "provider": "Z.ai", "acc": 51.6323, "ci": { "lo": 47.7203, "hi": 55.5443 }, "runs": 3, "trials": 1164, "cost": 0.4852, "lat": 281.6, "uncachedInput": 108334, "cachedTokens": 1056419, "recordedOutput": 13382 } }, { "rank": 2, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 51.5464, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 1.0179, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 422.6, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 145093, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1381795, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 11206, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "41.72–61.37%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 51.5464, "ci": { "lo": 41.722, "hi": 61.3708 }, "runs": 2, "trials": 776, "cost": 1.0179, "lat": 422.6, "uncachedInput": 145093, "cachedTokens": 1381795, "recordedOutput": 11206 } }, { "rank": 3, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 51.3746, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.5313, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 285.3, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 302342, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 2763672, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 25945, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "48.49–54.26%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 51.3746, "ci": { "lo": 48.4875, "hi": 54.2616 }, "runs": 3, "trials": 1164, "cost": 0.5313, "lat": 285.3, "uncachedInput": 302342, "cachedTokens": 2763672, "recordedOutput": 25945 } }, { "rank": 4, "ranked": true, "model": "Qwen 3.8 Max", "provider": "Alibaba", "metrics": { "score": { "value": 51.2887, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.8748, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 293.6, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 202606, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1483137, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 16464, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "46.81–55.77%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Qwen 3.8 Max", "provider": "Alibaba", "acc": 51.2887, "ci": { "lo": 46.8069, "hi": 55.7704 }, "runs": 3, "trials": 1164, "cost": 0.8748, "lat": 293.6, "uncachedInput": 202606, "cachedTokens": 1483137, "recordedOutput": 16464 } }, { "rank": 5, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 50.6873, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 1.3241, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 397.9, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 174545, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1905867, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 3675, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "45.09–56.28%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 50.6873, "ci": { "lo": 45.0935, "hi": 56.2811 }, "runs": 3, "trials": 1164, "cost": 1.3241, "lat": 397.9, "uncachedInput": 174545, "cachedTokens": 1905867, "recordedOutput": 3675 } }, { "rank": 6, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 49.0979, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 2.4747, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 536.9, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 96097, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 2126013, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 37248, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "37.64–60.56%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 49.0979, "ci": { "lo": 37.6362, "hi": 60.5597 }, "runs": 2, "trials": 776, "cost": 2.4747, "lat": 536.9, "uncachedInput": 96097, "cachedTokens": 2126013, "recordedOutput": 37248 } }, { "rank": 7, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 44.8454, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 1.805, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 410.6, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 40253, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 620696, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 15636, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "42.70–46.99%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 44.8454, "ci": { "lo": 42.7013, "hi": 46.9894 }, "runs": 4, "trials": 1552, "cost": 1.805, "lat": 410.6, "uncachedInput": 40253, "cachedTokens": 620696, "recordedOutput": 15636 } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 42.6546, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.3186, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 506.4, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 92762, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1289019, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 35220, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "37.74–47.57%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "acc": 42.6546, "ci": { "lo": 37.7424, "hi": 47.5668 }, "runs": 2, "trials": 776, "cost": 0.3186, "lat": 506.4, "uncachedInput": 92762, "cachedTokens": 1289019, "recordedOutput": 35220 } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 35.7388, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.0455, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 330.7, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 78687, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 936985, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 32785, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "32.58–38.90%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "acc": 35.7388, "ci": { "lo": 32.5806, "hi": 38.8971 }, "runs": 3, "trials": 1164, "cost": 0.0455, "lat": 330.7, "uncachedInput": 78687, "cachedTokens": 936985, "recordedOutput": 32785 } }, { "rank": 10, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 35.6529, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 1.1411, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 334, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 90188, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1257041, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 13879, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "33.59–37.71%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 35.6529, "ci": { "lo": 33.5948, "hi": 37.711 }, "runs": 3, "trials": 1164, "cost": 1.1411, "lat": 334, "uncachedInput": 90188, "cachedTokens": 1257041, "recordedOutput": 13879 } }, { "rank": 11, "ranked": true, "model": "Muse Spark 1.2", "provider": "Meta", "metrics": { "score": { "value": 34.2784, "unit": "percent", "label": "Score (mean trial reward)", "source_key": "acc" }, "cost_per_trial": { "value": 0.2977, "unit": "USD", "label": "Cost / trial", "source_key": "cost" }, "duration_per_trial": { "value": 228.4, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "uncachedInput": { "value": 44440, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "cachedTokens": { "value": 1348467, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 9382, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Muse Spark 1.2", "provider": "Meta", "acc": 34.2784, "ci": null, "runs": 1, "trials": 388, "cost": 0.2977, "lat": 228.4, "uncachedInput": 44440, "cachedTokens": 1348467, "recordedOutput": 9382 } } ] }, { "id": "customer-support/voice", "name": "τ³-Voice", "domain": "Customer Support", "industry": "Voice Agents", "contributor": "Sierra", "description": "Full-duplex voice agents evaluated on task completion and natural interaction across retail, airline, and telecom support.", "task_count": null, "status": "published", "headline_metric": { "key": "pass1", "label": "Score (Pass@1)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": "https://sierra.ai", "source_title": "Open Sierra", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-customer-support%2Fvoice", "methodology_summary": "Sierra’s full-duplex voice benchmark, published as τ-voice. The Specialized Intelligence Index displays a selected comparison set evaluated with the standard v1.0 user simulator; Fireworks does not rerun submissions.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "gpt-live-1", "provider": "OpenAI · backend: gpt-6 astra (medium) · v1.0", "metrics": { "score": { "value": 81.7193, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2.538, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "gpt-live-1-diamond-alpha_openai_2026-09-09", "model": "gpt-live-1", "organization": "OpenAI", "reasoning": "backend: gpt-6 astra (medium)", "userSim": "v1.0", "submissionType": "standard", "provider": "OpenAI · backend: gpt-6 astra (medium) · v1.0", "retail": 78.94736842105263, "airline": 82, "telecom": 84.21052631578947, "pass1": 81.71929824561403, "acc": 81.71929824561403, "responsiveness": 95.50621070472367, "latency": 2.5380343047723932, "lat": 2.5380343047723932, "cost": null, "interrupts": 21.970602710597813, "selectivity": 67.81573920496383 } }, { "rank": 2, "ranked": true, "model": "Pine Voice Preview", "provider": "Pine AI · enabled · v1.0", "metrics": { "score": { "value": 75.3801, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2.137, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "pine-voice-preview-user-sim-v1-0_pineai_2026-08-17", "model": "Pine Voice Preview", "organization": "Pine AI", "reasoning": "enabled", "userSim": "v1.0", "submissionType": "standard", "provider": "Pine AI · enabled · v1.0", "retail": 70.17543859649122, "airline": 70, "telecom": 85.96491228070175, "pass1": 75.38011695906432, "acc": 75.38011695906432, "responsiveness": 92.68629617177379, "latency": 2.1367219316875303, "lat": 2.1367219316875303, "cost": null, "interrupts": 46.88054814170788, "selectivity": 66.50371649329013 } }, { "rank": 3, "ranked": true, "model": "grok-voice-think-fast-1.0", "provider": "xAI · enabled · v1.0", "metrics": { "score": { "value": 67.3216, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.214, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "grok-voice-think-fast-1-0_xai_2026-04-21", "model": "grok-voice-think-fast-1.0", "organization": "xAI", "reasoning": "enabled", "userSim": "v1.0", "submissionType": "standard", "provider": "xAI · enabled · v1.0", "retail": 62.28070175438597, "airline": 66, "telecom": 73.68421052631578, "pass1": 67.32163742690058, "acc": 67.32163742690058, "responsiveness": 99.95008238473027, "latency": 1.21374532409684, "lat": 1.21374532409684, "cost": null, "interrupts": 19.72809650404285, "selectivity": 51.548727285569505 } }, { "rank": 4, "ranked": true, "model": "grok-voice-think-fast-2.0", "provider": "xAI · high · v1.0", "metrics": { "score": { "value": 62.5263, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.708, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "grok-voice-think-fast-2-0_xai_2026-08-06", "model": "grok-voice-think-fast-2.0", "organization": "xAI", "reasoning": "high", "userSim": "v1.0", "submissionType": "standard", "provider": "xAI · high · v1.0", "retail": 59.64912280701754, "airline": 56.00000000000001, "telecom": 71.9298245614035, "pass1": 62.52631578947368, "acc": 62.52631578947368, "responsiveness": 99.96831432192648, "latency": 1.7084125004062407, "lat": 1.7084125004062407, "cost": null, "interrupts": 10.683453861687237, "selectivity": 35.93217654410495 } }, { "rank": 5, "ranked": true, "model": "qwen3.5-omni-plus-realtime", "provider": "Qwen · — · v1.0", "metrics": { "score": { "value": 53.6725, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.698, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "qwen3-5-omni-plus-realtime_qwen_2026-08-06", "model": "qwen3.5-omni-plus-realtime", "organization": "Qwen", "reasoning": "—", "userSim": "v1.0", "submissionType": "standard", "provider": "Qwen · — · v1.0", "retail": 46.49122807017544, "airline": 54, "telecom": 60.526315789473685, "pass1": 53.67251461988304, "acc": 53.67251461988304, "responsiveness": 99.1876465990297, "latency": 1.6983165562514282, "lat": 1.6983165562514282, "cost": null, "interrupts": 36.11213067271342, "selectivity": 44.95802300083696 } }, { "rank": 6, "ranked": true, "model": "gemini-3.1-flash-live-preview-thinking-high", "provider": "Google · high · v1.0", "metrics": { "score": { "value": 43.848, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 3.15, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "gemini-3-1-flash-live-preview-thinking-high_google_2026-04-02", "model": "gemini-3.1-flash-live-preview-thinking-high", "organization": "Google", "reasoning": "high", "userSim": "v1.0", "submissionType": "standard", "provider": "Google · high · v1.0", "retail": 45.614035087719294, "airline": 64, "telecom": 21.929824561403507, "pass1": 43.847953216374265, "acc": 43.847953216374265, "responsiveness": 84.71056287721113, "latency": 3.1501594263481407, "lat": 3.1501594263481407, "cost": null, "interrupts": 18.854275199400824, "selectivity": 66.29082340489443 } }, { "rank": 7, "ranked": true, "model": "gpt-realtime-2", "provider": "OpenAI · xhigh · v1.0", "metrics": { "score": { "value": 42.4327, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.978, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "gpt-realtime-2_openai_2026-06-29", "model": "gpt-realtime-2", "organization": "OpenAI", "reasoning": "xhigh", "userSim": "v1.0", "submissionType": "standard", "provider": "OpenAI · xhigh · v1.0", "retail": 47.368421052631575, "airline": 57.99999999999999, "telecom": 21.929824561403507, "pass1": 42.43274853801169, "acc": 42.43274853801169, "responsiveness": 95.10025830375763, "latency": 1.9783190850360786, "lat": 1.9783190850360786, "cost": null, "interrupts": 20.759755817025013, "selectivity": 10.757638654832446 } }, { "rank": 8, "ranked": true, "model": "gpt-realtime-2", "provider": "OpenAI · minimal · v1.0", "metrics": { "score": { "value": 38.5497, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.44, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "gpt-realtime-2-minimal_openai_2026-07-01", "model": "gpt-realtime-2", "organization": "OpenAI", "reasoning": "minimal", "userSim": "v1.0", "submissionType": "standard", "provider": "OpenAI · minimal · v1.0", "retail": 39.473684210526315, "airline": 56.00000000000001, "telecom": 20.175438596491226, "pass1": 38.54970760233919, "acc": 38.54970760233919, "responsiveness": 99.80739207280237, "latency": 1.4395493876335355, "lat": 1.4395493876335355, "cost": null, "interrupts": 18.679347068488358, "selectivity": 8.333117770196461 } }, { "rank": 9, "ranked": true, "model": "grok-voice-fast-1.0", "provider": "xAI · — · v1.0", "metrics": { "score": { "value": 38.3333, "unit": "percent", "label": "Score (Pass@1)", "source_key": "pass1" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1.149, "unit": "seconds", "label": "Duration / task", "source_key": "lat" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "submissionId": "xai-realtime_sierra_2026-03-03", "model": "grok-voice-fast-1.0", "organization": "xAI", "reasoning": "—", "userSim": "v1.0", "submissionType": "standard", "provider": "xAI · — · v1.0", "retail": 38.6, "airline": 36, "telecom": 40.4, "pass1": 38.333333333333336, "acc": 38.333333333333336, "responsiveness": 91.3223367979192, "latency": 1.1492654229568544, "lat": 1.1492654229568544, "cost": null, "interrupts": 84.29014444073495, "selectivity": 57.458909085822974 } } ] }, { "id": "code-generation/frontierswe-v2", "name": "FrontierSWE V2", "domain": "Software", "industry": "Code Generation", "contributor": "Proximal", "description": "Thirty-four software-engineering tasks pitched at the edge of what an expert human can do: writing a flight-sim renderer in OpenGL, porting Git to Zig, driving a racing bot from vision alone. Each model gets five trials per task and up to twenty hours per trial, and every trial earns a graded reward rather than a pass or a fail, so a run that gets most of the way there still scores.", "task_count": 34, "status": "published", "headline_metric": { "key": "acc", "label": "Score (mean@5)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://www.frontierswe.com/", "source_title": "Open FrontierSWE", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-code-generation%2Ffrontierswe-v2", "methodology_summary": "FrontierSWE V2 measures software-engineering skill on 34 tasks pitched at the edge of expert human ability, scored by graded per-task reward rather than a binary verdict. The Specialized Intelligence Index reproduces Proximal’s own published mean@5 leaderboard without rerunning it.", "board_note": null, "result_count": 10, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 65.5, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 5148.25, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 43560, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 1029.65, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 8.9, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": null, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": null, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 65.5, "publishedSpread": 8.9, "cost": 5148.25, "costPerAttempt": 1029.65, "lat": 43560, "tokensPerTrial": null, "stepsPerTrial": null, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 2, "ranked": true, "model": "Claude Fable 5.1", "provider": "Anthropic", "metrics": { "score": { "value": 56.29, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 692.75, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 41760, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 138.55, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 11.1, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 246000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 623, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Fable 5.1", "provider": "Anthropic", "acc": 56.28999999999999, "publishedSpread": 11.1, "cost": 692.75, "costPerAttempt": 138.55, "lat": 41760, "tokensPerTrial": 246000000, "stepsPerTrial": 623, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 3, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 52.01, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 983.55, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 57960, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 196.71, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 10.8, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": null, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": null, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 52.01, "publishedSpread": 10.8, "cost": 983.5500000000001, "costPerAttempt": 196.71, "lat": 57960.00000000001, "tokensPerTrial": null, "stepsPerTrial": null, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 4, "ranked": true, "model": "GPT-5.6", "provider": "OpenAI", "metrics": { "score": { "value": 32.2, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 898.2, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 30960, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 179.64, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 11, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 182000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 533, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6", "provider": "OpenAI", "acc": 32.2, "publishedSpread": 11, "cost": 898.1999999999999, "costPerAttempt": 179.64, "lat": 30960, "tokensPerTrial": 182000000, "stepsPerTrial": 533, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 5, "ranked": true, "model": "GLM-5.3", "provider": "Z.ai", "metrics": { "score": { "value": 30.18, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 486.1, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 61200, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 97.22, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 11.5, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 333000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 808, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GLM-5.3", "provider": "Z.ai", "acc": 30.18, "publishedSpread": 11.5, "cost": 486.1, "costPerAttempt": 97.22, "lat": 61200, "tokensPerTrial": 333000000, "stepsPerTrial": 808, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 6, "ranked": true, "model": "Grok 4.7", "provider": "xAI", "metrics": { "score": { "value": 29.5, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 1593.85, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 43560, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 318.77, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 12.6, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": null, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": null, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.7", "provider": "xAI", "acc": 29.5, "publishedSpread": 12.6, "cost": 1593.85, "costPerAttempt": 318.77, "lat": 43560, "tokensPerTrial": null, "stepsPerTrial": null, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 7, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 25.87, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 548.55, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 66240, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 109.71, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 11.8, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 304000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 763, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 25.869999999999997, "publishedSpread": 11.8, "cost": 548.55, "costPerAttempt": 109.71, "lat": 66240, "tokensPerTrial": 304000000, "stepsPerTrial": 763, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 8, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 25.29, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 1217.15, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 49680, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 243.43, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 12.2, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 229000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 882, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 25.290000000000003, "publishedSpread": 12.2, "cost": 1217.15, "costPerAttempt": 243.43, "lat": 49680, "tokensPerTrial": 229000000, "stepsPerTrial": 882, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 9, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 19.63, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 193.75, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 25920, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 38.75, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 11.1, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": null, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": null, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 19.63, "publishedSpread": 11.1, "cost": 193.75, "costPerAttempt": 38.75, "lat": 25920, "tokensPerTrial": null, "stepsPerTrial": null, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } }, { "rank": 10, "ranked": true, "model": "DeepSeek V4 Flash Vision Exp", "provider": "DeepSeek", "metrics": { "score": { "value": 14.8, "unit": "percent", "label": "Score (mean@5)", "source_key": "acc" }, "cost_per_task": { "value": 42.85, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_trial": { "value": 53280, "unit": "seconds", "label": "Duration / trial", "source_key": "lat" }, "costPerAttempt": { "value": 8.57, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 5, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 170, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 170, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "publishedSpread": { "value": 9.7, "unit": "percentage points", "label": "Worst@5–best@5 spread", "source_key": "publishedSpread" }, "tokensPerTrial": { "value": 513000000, "unit": "tokens", "label": "Average tokens / trial", "source_key": "tokensPerTrial" }, "stepsPerTrial": { "value": 1210, "unit": "count", "label": "Average steps / trial", "source_key": "stepsPerTrial" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": "170 / 170 (100.00%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Flash Vision Exp", "provider": "DeepSeek", "acc": 14.799999999999999, "publishedSpread": 9.7, "cost": 42.85, "costPerAttempt": 8.57, "lat": 53280, "tokensPerTrial": 513000000, "stepsPerTrial": 1210, "k": 5, "attemptsScored": 170, "attemptsExpected": 170, "coverage": "170 / 170 (100.00%)" } } ] }, { "id": "code-generation/apex-swe", "name": "APEX-SWE", "domain": "Software", "industry": "Code Generation", "contributor": "Mercor", "description": "Repository-scale engineering tasks graded against the patch a maintainer would ship.", "task_count": 200, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@4)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://www.mercor.com/apex/apex-swe-leaderboard/", "source_title": "Open Mercor APEX-SWE", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-code-generation%2Fapex-swe", "methodology_summary": "APEX-SWE evaluates integration and observability engineering in containerized repositories with programmatic verification. The Specialized Intelligence Index publishes Mercor’s supplied four-attempt Pass@1 scores on the complete 200-task held-out set.", "board_note": null, "result_count": 8, "results": [ { "rank": 1, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 63.75, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 21.931, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 63.75, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 5.4829, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 3182369, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "57.50–70.00%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "servedBy": "Anthropic", "modelIdentifier": "claude-opus-5", "reasoning": "max", "acc": 63.75, "ci": { "lo": 57.5, "hi": 70, "halfWidth": 6.25 }, "pass1": 63.75, "pass1CiHalfWidth": 6.25, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 21.931, "costPerAttempt": 5.4829, "avgTokens": 3182369, "k": 4, "priceBasis": "anthropic/claude-opus-5", "batchId": "batch_8f821777fdd24137a64f36c0f5a26ef6" } }, { "rank": 2, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 57.75, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 8.386, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 57.75, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.12, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 2.0966, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2509336, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "51.63–63.87%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "servedBy": "xAI", "modelIdentifier": "grok-4.6", "reasoning": "xhigh", "acc": 57.75, "ci": { "lo": 51.63, "hi": 63.87, "halfWidth": 6.12 }, "pass1": 57.75, "pass1CiHalfWidth": 6.12, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 8.386, "costPerAttempt": 2.0966, "avgTokens": 2509336, "k": 4, "priceBasis": "xai/grok-4.6", "batchId": "batch_61ac1c3094f44cd19f087903ca48bf54" } }, { "rank": 3, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 51.88, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.978, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 51.88, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.2444, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2620581, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "45.63–58.13%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4p1-flash", "reasoning": "max", "acc": 51.88, "ci": { "lo": 45.63, "hi": 58.13, "halfWidth": 6.25 }, "pass1": 51.88, "pass1CiHalfWidth": 6.25, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 0.978, "costPerAttempt": 0.2444, "avgTokens": 2620581, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4p1-flash", "batchId": "batch_488cd1ae34f74539a6cb23fb4996a6c4" } }, { "rank": 4, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 50, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 54.889, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 50, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.44, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 13.7222, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 1179698, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "43.56–56.44%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "servedBy": "OpenAI", "modelIdentifier": "gpt-6-astra", "reasoning": "max", "acc": 50, "ci": { "lo": 43.56, "hi": 56.44, "halfWidth": 6.44 }, "pass1": 50, "pass1CiHalfWidth": 6.44, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 54.889, "costPerAttempt": 13.7222, "avgTokens": 1179698, "k": 4, "priceBasis": "openai/gpt-6-astra", "batchId": "batch_c1c14152604146d6b29b0eeae3fe4387" } }, { "rank": 5, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 48, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 2.589, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 48, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.6472, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 815738, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "41.75–54.25%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "servedBy": "Fireworks", "modelIdentifier": "accounts/harshvardhan-660c0e/routers/mike-mtp-mercor", "reasoning": "max", "acc": 48, "ci": { "lo": 41.75, "hi": 54.25, "halfWidth": 6.25 }, "pass1": 48, "pass1CiHalfWidth": 6.25, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 2.589, "costPerAttempt": 0.6472, "avgTokens": 815738, "k": 4, "priceBasis": "fireworks_ai/.../kimi-k3", "batchId": "batch_05b997c0753d4a51a38db18f6fe13ec6" } }, { "rank": 6, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 46.92, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 8.045, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 46.92, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6.21, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 2.0137, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 799, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2175569, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "40.71–53.13%", "per_run_scores": null, "coverage": "799 / 800 (99.88%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4-pro-0813", "reasoning": "max", "acc": 46.92, "ci": { "lo": 40.71, "hi": 53.13, "halfWidth": 6.21 }, "pass1": 46.92, "pass1CiHalfWidth": 6.21, "attemptsScored": 799, "attemptsExpected": 800, "coverage": "799 / 800 (99.88%)", "cost": 8.045, "costPerAttempt": 2.0137, "avgTokens": 2175569, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4-pro-0813", "batchId": "batch_a08835312bac4dd0b784a479ee034224" } }, { "rank": 7, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 45.62, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 12.777, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 45.62, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 5.94, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 3.1943, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 800, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 2596909, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "39.68–51.56%", "per_run_scores": null, "coverage": "800 / 800 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/glm-5p3", "reasoning": "max", "acc": 45.62, "ci": { "lo": 39.68, "hi": 51.56, "halfWidth": 5.94 }, "pass1": 45.62, "pass1CiHalfWidth": 5.94, "attemptsScored": 800, "attemptsExpected": 800, "coverage": "800 / 800 (100%)", "cost": 12.777, "costPerAttempt": 3.1943, "avgTokens": 2596909, "k": 4, "priceBasis": "fireworks_ai/.../glm-5p3", "batchId": "batch_7f7badeba05c434589888a97cdade21d" } }, { "rank": 8, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 36.62, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 3.774, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 36.62, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 6, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.9482, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 796, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 800, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 4365138, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "30.62–42.62%", "per_run_scores": null, "coverage": "796 / 800 (99.5%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "servedBy": "Google Vertex", "modelIdentifier": "gemini-3.8-flash", "reasoning": "high", "acc": 36.62, "ci": { "lo": 30.62, "hi": 42.62, "halfWidth": 6 }, "pass1": 36.62, "pass1CiHalfWidth": 6, "attemptsScored": 796, "attemptsExpected": 800, "coverage": "796 / 800 (99.5%)", "cost": 3.774, "costPerAttempt": 0.9482, "avgTokens": 4365138, "k": 4, "priceBasis": "vertex_ai/gemini-3.8-flash", "batchId": "batch_41ec461cbc0045cdba3102b201fcf9a7" } } ] }, { "id": "code-generation/deepswe-v1-1", "name": "DeepSWE v1.1", "domain": "Software", "industry": "Code Generation", "contributor": null, "description": "Deep debugging tasks where the failure cause sits several layers from the symptom.", "task_count": 113, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": null, "source_title": null, "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-code-generation%2Fdeepswe-v1-1", "methodology_summary": "Original, long-horizon software-engineering tasks in real repositories, verified by held-out tests.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 72.27, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 6.1774, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1309, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 137428, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 7469471, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 7332043, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 66103, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "69.96–74.58%", "per_run_scores": [ 69.91, 73.45, 73.45 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 7469471, "model": "Claude Opus 5", "provider": "Anthropic", "runs": [ 69.91, 73.45, 73.45 ], "cost": 6.1774, "lat": 1309, "uncachedInput": 137428, "cachedTokens": 7332043, "recordedOutput": 66103, "acc": 72.27, "runScores": "69.91 · 73.45 · 73.45", "ci": { "lo": 69.95724247999999, "hi": 74.58275752, "halfWidth": 2.3127575200000043, "runs": 3 } } }, { "rank": 2, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 71.01, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 2.559, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 681, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 60107, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 881158, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 821051, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 19746, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "68.07–73.95%", "per_run_scores": [ 73.21, 71.68, 68.14 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 881158, "model": "GPT-6 Astra", "provider": "OpenAI", "runs": [ 73.21, 71.68, 68.14 ], "cost": 2.559, "lat": 681, "uncachedInput": 60107, "cachedTokens": 821051, "recordedOutput": 19746, "acc": 71.00999999999999, "runScores": "73.21 · 71.68 · 68.14", "ci": { "lo": 68.06724605919801, "hi": 73.95275394080197, "halfWidth": 2.9427539408019756, "runs": 3 } } }, { "rank": 3, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 70.21, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 5.3971, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2340, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 190672, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 11799728, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 11609056, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 89493, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "69.05–71.37%", "per_run_scores": [ 70.8, 69.03, 70.8 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 11799728, "model": "Kimi K3", "provider": "Moonshot AI", "runs": [ 70.8, 69.03, 70.8 ], "cost": 5.3971, "lat": 2340, "uncachedInput": 190672, "cachedTokens": 11609056, "recordedOutput": 89493, "acc": 70.21, "runScores": "70.80 · 69.03 · 70.80", "ci": { "lo": 69.05362124, "hi": 71.36637875999999, "halfWidth": 1.1563787599999975, "runs": 3 } } }, { "rank": 4, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 68.1433, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 4.7258, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2294, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 270251, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 15351177, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 15080926, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 96907, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "63.56–72.73%", "per_run_scores": [ 71.68, 69.03, 63.72 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 15351177, "model": "GLM 5.3", "provider": "Z.ai", "runs": [ 71.68, 69.03, 63.72 ], "cost": 4.7258, "lat": 2294, "uncachedInput": 270251, "cachedTokens": 15080926, "recordedOutput": 96907, "acc": 68.14333333333333, "runScores": "71.68 · 69.03 · 63.72", "ci": { "lo": 63.55656622681474, "hi": 72.73010043985192, "halfWidth": 4.586767106518591, "runs": 3 } } }, { "rank": 5, "ranked": true, "model": "DeepSeek v4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 63.4233, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.9177, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1317, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 2479261, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 13063811, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 10584550, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 91982, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "57.73–69.12%", "per_run_scores": [ 69.03, 61.95, 59.29 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 13063811, "model": "DeepSeek v4.1 Flash", "provider": "DeepSeek", "runs": [ 69.03, 61.95, 59.29 ], "cost": 0.9177, "lat": 1317, "uncachedInput": 2479261, "cachedTokens": 10584550, "recordedOutput": 91982, "acc": 63.42333333333334, "runScores": "69.03 · 61.95 · 59.29", "ci": { "lo": 57.726506168171184, "hi": 69.1201604984955, "halfWidth": 5.696827165162154, "runs": 3 } } }, { "rank": 6, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 61.95, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 2.0106, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 545, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": null, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 1517186, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": null, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 20025, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "59.95–63.95%", "per_run_scores": [ 60.18, 61.95, 63.72 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 1517186, "model": "GPT-5.6 Sol", "provider": "OpenAI", "runs": [ 60.18, 61.95, 63.72 ], "cost": 2.0106, "lat": 545, "uncachedInput": null, "cachedTokens": null, "cacheRecorded": false, "recordedOutput": 20025, "acc": 61.949999999999996, "runScores": "60.18 · 61.95 · 63.72", "ci": { "lo": 59.9470932348865, "hi": 63.95290676511349, "halfWidth": 2.0029067651134964, "runs": 3 } } }, { "rank": 7, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 61.65, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 4.2422, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1180, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 1254164, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 30839486, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 29585322, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 288712, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "56.61–66.69%", "per_run_scores": [ 61.06, 57.52, 66.37 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 30839486, "model": "Gemini 3.8 Flash", "provider": "Google", "runs": [ 61.06, 57.52, 66.37 ], "cost": 4.2422, "lat": 1180, "uncachedInput": 1254164, "cachedTokens": 29585322, "recordedOutput": 288712, "acc": 61.650000000000006, "runScores": "61.06 · 57.52 · 66.37", "ci": { "lo": 56.609461844703134, "hi": 66.69053815529688, "halfWidth": 5.0405381552968755, "runs": 3 } } }, { "rank": 8, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 57.8167, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 6.0288, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 5240, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 653412, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 6378373, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 5724961, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 309918, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "57.24–58.40%", "per_run_scores": [ 57.52, 58.41, 57.52 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 6378373, "model": "Grok 4.6", "provider": "xAI", "runs": [ 57.52, 58.41, 57.52 ], "cost": 6.0288, "lat": 5240, "uncachedInput": 653412, "cachedTokens": 5724961, "recordedOutput": 309918, "acc": 57.81666666666667, "runScores": "57.52 · 58.41 · 57.52", "ci": { "lo": 57.23521068000001, "hi": 58.39812265333333, "halfWidth": 0.5814559866666623, "runs": 3 } } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 45.13, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 1.1231, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1264, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "uncachedInput": { "value": 243277, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 11359099, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 11115822, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 79019, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" } }, "confidence_interval_95": "43.13–47.13%", "per_run_scores": [ 43.36, 46.9, 45.13 ], "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "inputTokens": 11359099, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "runs": [ 43.36, 46.9, 45.13 ], "cost": 1.1231, "lat": 1264, "uncachedInput": 243277, "cachedTokens": 11115822, "recordedOutput": 79019, "acc": 45.129999999999995, "runScores": "43.36 · 46.90 · 45.13", "ci": { "lo": 43.1270932348865, "hi": 47.13290676511349, "halfWidth": 2.0029067651134964, "runs": 3 } } } ] }, { "id": "code-review/macroscope-code-review-benchmark", "name": "MacroscopeBench", "domain": "Software", "industry": "Code Review", "contributor": "Macroscope", "description": "MacroscopeBench evaluates a model’s ability to identify bugs in real code changes while avoiding incorrect bug reports. Each task reviews a commit from a public repository; the evaluation includes commits containing known, subsequently fixed bugs and control commits with no known bugs recorded in the dataset. LLM judges determine whether the model identified the known bugs and assess the validity of all reported findings. The performance of a model is scored based on the harmonic mean of the known bug recall and overall bug detection precision as evaluated by the LLM judge.", "task_count": 195, "status": "published", "headline_metric": { "key": "f1Score", "label": "Score (avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://macroscope.com/benchmark", "source_title": "Open MacroscopeBench", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-code-review%2Fmacroscope-code-review-benchmark", "methodology_summary": "MacroscopeBench evaluates a model’s ability to identify bugs in real code changes while avoiding incorrect bug reports. Each task reviews a commit from a public repository; the evaluation includes commits containing known, subsequently fixed bugs and control commits with no known bugs recorded in the dataset. LLM judges determine whether the model identified the known bugs and assess the validity of all reported findings. The performance of a model is scored based on the harmonic mean of the known bug recall and overall bug detection precision as evaluated by the LLM judge.", "board_note": null, "result_count": 21, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra · max", "provider": "OpenAI", "metrics": { "score": { "value": 77.9611, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 7.8746, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 228.86, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 74.3056, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 67.8241, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 91.6606, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 45.2381, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 70.4545, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 77.5758, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 56.9892, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 6.2565, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 478.725, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 935.58, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "astra-max-control", "model": "GPT-6 Astra · max", "provider": "OpenAI", "effort": "max", "k": 3, "acc": 77.9611, "f1Score": 77.9611, "cost": 7.874571, "costPerReview": 7.874571, "lat": 228.86, "latencyMsP50": 228860, "recallAvgK": 67.8241, "precisionAvgK": 91.6606, "recallCritical": 45.2381, "recallHigh": 70.4545, "recallMedium": 77.5758, "recallLow": 56.9892, "recallMaxK": 74.3056, "snr": 6.256545, "latencyP90": 478.725, "latencyMax": 935.58, "totalCost": 4606.62, "rankEligible": true, "benchmarkId": "a44b3d47-5e28-5fc1-8b85-1425b991ae62", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.779611, "recallAvgK": 0.678241, "recallMaxK": 0.743056, "precisionAvgK": 0.916606, "signalToNoiseRatio": 6.256545, "bySeverity": { "critical": { "recall": 0.452381, "precision": 0.950617, "precisionScored": 81 }, "high": { "recall": 0.704545, "precision": 0.912525, "precisionScored": 1509 }, "low": { "recall": 0.569892, "precision": 0.869565, "precisionScored": 161 }, "medium": { "recall": 0.775758, "precision": 0.928717, "precisionScored": 982 }, "none": { "recall": null, "precision": 0.88, "precisionScored": 25 } }, "totalCost": 4606.62, "costPerReview": 7.874571, "latencyMsP50": 228860, "latencyMsP90": 478725, "latencyMsMax": 935580, "effectiveModel": { "provider": "openai", "modelName": "gpt-6-astra", "effort": "max", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GPT-6 Astra", "displayModel": "GPT-6 Astra · max" } }, { "rank": 2, "ranked": true, "model": "GPT-5.6 Sol · high", "provider": "OpenAI", "metrics": { "score": { "value": 77.7682, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 3.9031, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 180.398, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 80.5556, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 70.8333, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 86.2083, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 50, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 69.697, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 77.5758, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 69.8925, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 3.7326, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 330.454, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 1505.101, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "sol-high-control", "model": "GPT-5.6 Sol · high", "provider": "OpenAI", "effort": "high", "k": 3, "acc": 77.7682, "f1Score": 77.7682, "cost": 3.903123, "costPerReview": 3.903123, "lat": 180.398, "latencyMsP50": 180398, "recallAvgK": 70.8333, "precisionAvgK": 86.2083, "recallCritical": 50, "recallHigh": 69.697, "recallMedium": 77.5758, "recallLow": 69.8925, "recallMaxK": 80.5556, "snr": 3.732558, "latencyP90": 330.454, "latencyMax": 1505.101, "totalCost": 2283.33, "rankEligible": true, "benchmarkId": "909a01d6-c340-5b97-9367-68493498325a", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.777682, "recallAvgK": 0.708333, "recallMaxK": 0.805556, "precisionAvgK": 0.862083, "signalToNoiseRatio": 3.732558, "bySeverity": { "critical": { "recall": 0.5, "precision": 1, "precisionScored": 56 }, "high": { "recall": 0.69697, "precision": 0.886047, "precisionScored": 1290 }, "low": { "recall": 0.698925, "precision": 0.834862, "precisionScored": 218 }, "medium": { "recall": 0.775758, "precision": 0.85875, "precisionScored": 800 }, "none": { "recall": null, "precision": 0.4, "precisionScored": 65 } }, "totalCost": 2283.33, "costPerReview": 3.903123, "latencyMsP50": 180398, "latencyMsP90": 330454, "latencyMsMax": 1505101, "effectiveModel": { "provider": "openai", "modelName": "gpt-5.6-sol", "effort": "high", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GPT-5.6 Sol", "displayModel": "GPT-5.6 Sol · high" } }, { "rank": 3, "ranked": true, "model": "GLM 5.3 · max", "provider": "Z.ai", "metrics": { "score": { "value": 77.379, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 3.8585, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 750.345, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 87.5, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 76.3889, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 78.3951, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 69.0476, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 75, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 83.0303, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 69.8925, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 2.2392, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 1367.982, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 2157.653, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "glm53-max-control", "model": "GLM 5.3 · max", "provider": "Z.ai", "effort": "max", "k": 3, "acc": 77.379, "f1Score": 77.379, "cost": 3.85854, "costPerReview": 3.85854, "lat": 750.345, "latencyMsP50": 750345, "recallAvgK": 76.3889, "precisionAvgK": 78.3951, "recallCritical": 69.0476, "recallHigh": 75, "recallMedium": 83.0303, "recallLow": 69.8925, "recallMaxK": 87.5, "snr": 2.239156, "latencyP90": 1367.982, "latencyMax": 2157.653, "totalCost": 2257.25, "rankEligible": true, "benchmarkId": "f8ed8869-c28f-5bba-a5c5-6e366685d145", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.77379, "recallAvgK": 0.763889, "recallMaxK": 0.875, "precisionAvgK": 0.783951, "signalToNoiseRatio": 2.239156, "bySeverity": { "critical": { "recall": 0.690476, "precision": 0.926316, "precisionScored": 95 }, "high": { "recall": 0.75, "precision": 0.810589, "precisionScored": 1341 }, "low": { "recall": 0.698925, "precision": 0.728571, "precisionScored": 490 }, "medium": { "recall": 0.830303, "precision": 0.81958, "precisionScored": 715 }, "none": { "recall": null, "precision": 0.362832, "precisionScored": 113 } }, "totalCost": 2257.25, "costPerReview": 3.85854, "latencyMsP50": 750345, "latencyMsP90": 1367982, "latencyMsMax": 2157653, "effectiveModel": { "provider": "fireworks", "modelName": "zhipuai/GLM-5.3", "effort": "max", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GLM 5.3", "displayModel": "GLM 5.3 · max" } }, { "rank": 4, "ranked": true, "model": "Grok 4.6 · xhigh", "provider": "xAI", "metrics": { "score": { "value": 75.8716, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 4.6818, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 880.604, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 78.4722, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 69.6759, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 83.2766, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 40.4762, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 68.9394, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 84.8485, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 56.9892, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 3.6658, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 2735.451, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 3578.916, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "grok46-xhigh-control", "model": "Grok 4.6 · xhigh", "provider": "xAI", "effort": "xhigh", "k": 3, "acc": 75.8716, "f1Score": 75.8716, "cost": 4.681813, "costPerReview": 4.681813, "lat": 880.604, "latencyMsP50": 880604, "recallAvgK": 69.6759, "precisionAvgK": 83.2766, "recallCritical": 40.4762, "recallHigh": 68.9394, "recallMedium": 84.8485, "recallLow": 56.9892, "recallMaxK": 78.4722, "snr": 3.665789, "latencyP90": 2735.451, "latencyMax": 3578.916, "totalCost": 2738.86, "rankEligible": true, "benchmarkId": "a46493ee-207c-5a4e-acc6-041c5a21cab4", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.758716, "recallAvgK": 0.696759, "recallMaxK": 0.784722, "precisionAvgK": 0.832766, "signalToNoiseRatio": 3.665789, "bySeverity": { "critical": { "recall": 0.404762, "precision": 0.894737, "precisionScored": 57 }, "high": { "recall": 0.689394, "precision": 0.86036, "precisionScored": 1110 }, "low": { "recall": 0.569892, "precision": 0.780488, "precisionScored": 82 }, "medium": { "recall": 0.848485, "precision": 0.84009, "precisionScored": 444 }, "none": { "recall": null, "precision": 0.366197, "precisionScored": 71 } }, "totalCost": 2738.86, "costPerReview": 4.681813, "latencyMsP50": 880604, "latencyMsP90": 2735451, "latencyMsMax": 3578916, "effectiveModel": { "provider": "xai", "modelName": "grok-4.6", "effort": "xhigh", "contextWindow": 500000 } }, "comparableAvg3": true, "baseModel": "Grok 4.6", "displayModel": "Grok 4.6 · xhigh" } }, { "rank": 5, "ranked": true, "model": "Grok 4.6 · high", "provider": "xAI", "metrics": { "score": { "value": 75.0868, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 3.5108, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 929.989, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 81.25, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 69.9074, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 81.0951, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 50, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 68.9394, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 83.0303, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 56.9892, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 3.1675, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 2577.037, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 3580.73, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "grok46-high-control", "model": "Grok 4.6 · high", "provider": "xAI", "effort": "high", "k": 3, "acc": 75.0868, "f1Score": 75.0868, "cost": 3.510833, "costPerReview": 3.510833, "lat": 929.989, "latencyMsP50": 929989, "recallAvgK": 69.9074, "precisionAvgK": 81.0951, "recallCritical": 50, "recallHigh": 68.9394, "recallMedium": 83.0303, "recallLow": 56.9892, "recallMaxK": 81.25, "snr": 3.167464, "latencyP90": 2577.037, "latencyMax": 3580.73, "totalCost": 2053.84, "rankEligible": true, "benchmarkId": "6c6bf706-ce17-5d5b-beb5-14e1ca274c77", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.750868, "recallAvgK": 0.699074, "recallMaxK": 0.8125, "precisionAvgK": 0.810951, "signalToNoiseRatio": 3.167464, "bySeverity": { "critical": { "recall": 0.5, "precision": 0.935484, "precisionScored": 62 }, "high": { "recall": 0.689394, "precision": 0.8327, "precisionScored": 1052 }, "low": { "recall": 0.569892, "precision": 0.732558, "precisionScored": 86 }, "medium": { "recall": 0.830303, "precision": 0.841758, "precisionScored": 455 }, "none": { "recall": null, "precision": 0.3375, "precisionScored": 80 } }, "totalCost": 2053.84, "costPerReview": 3.510833, "latencyMsP50": 929989, "latencyMsP90": 2577037, "latencyMsMax": 3580730, "effectiveModel": { "provider": "xai", "modelName": "grok-4.6", "effort": "high", "contextWindow": 500000 } }, "comparableAvg3": true, "baseModel": "Grok 4.6", "displayModel": "Grok 4.6 · high" } }, { "rank": 6, "ranked": true, "model": "Kimi K3 · max", "provider": "Moonshot AI", "metrics": { "score": { "value": 72.8593, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 2.8225, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 493.238, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 77.0833, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 66.6667, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 80.3201, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 47.619, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 65.9091, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 78.7879, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 54.8387, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 2.1811, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 1334.473, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 3407.996, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "k3-max-control", "model": "Kimi K3 · max", "provider": "Moonshot AI", "effort": "max", "k": 3, "acc": 72.8593, "f1Score": 72.8593, "cost": 2.822498, "costPerReview": 2.822498, "lat": 493.238, "latencyMsP50": 493238, "recallAvgK": 66.6667, "precisionAvgK": 80.3201, "recallCritical": 47.619, "recallHigh": 65.9091, "recallMedium": 78.7879, "recallLow": 54.8387, "recallMaxK": 77.0833, "snr": 2.18107, "latencyP90": 1334.473, "latencyMax": 3407.996, "totalCost": 1651.16, "rankEligible": true, "benchmarkId": "5b0f70ad-7c7d-5627-882b-1951e67fb1c1", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.728593, "recallAvgK": 0.666667, "recallMaxK": 0.770833, "precisionAvgK": 0.803201, "signalToNoiseRatio": 2.18107, "bySeverity": { "critical": { "recall": 0.47619, "precision": 0.887324, "precisionScored": 71 }, "high": { "recall": 0.659091, "precision": 0.857279, "precisionScored": 1051 }, "low": { "recall": 0.548387, "precision": 0.759834, "precisionScored": 483 }, "medium": { "recall": 0.787879, "precision": 0.831081, "precisionScored": 592 }, "none": { "recall": null, "precision": 0.295652, "precisionScored": 115 } }, "totalCost": 1651.16, "costPerReview": 2.822498, "latencyMsP50": 493238, "latencyMsP90": 1334473, "latencyMsMax": 3407996, "effectiveModel": { "provider": "fireworks", "modelName": "moonshotai/Kimi-K3", "effort": "max", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Kimi K3", "displayModel": "Kimi K3 · max" } }, { "rank": 7, "ranked": true, "model": "Grok 4.6 · medium", "provider": "xAI", "metrics": { "score": { "value": 72.8008, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 2.5609, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 518.544, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 75.6944, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 67.1296, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 79.5187, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 50, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 65.9091, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 83.0303, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 48.3871, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 3.1124, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 2088.625, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 3529.837, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "grok46-med-control", "model": "Grok 4.6 · medium", "provider": "xAI", "effort": "medium", "k": 3, "acc": 72.8008, "f1Score": 72.8008, "cost": 2.560935, "costPerReview": 2.560935, "lat": 518.544, "latencyMsP50": 518544, "recallAvgK": 67.1296, "precisionAvgK": 79.5187, "recallCritical": 50, "recallHigh": 65.9091, "recallMedium": 83.0303, "recallLow": 48.3871, "recallMaxK": 75.6944, "snr": 3.112385, "latencyP90": 2088.625, "latencyMax": 3529.837, "totalCost": 1498.15, "rankEligible": true, "benchmarkId": "cb59e9a8-c128-56c1-ace9-2f291b18f118", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.728008, "recallAvgK": 0.671296, "recallMaxK": 0.756944, "precisionAvgK": 0.795187, "signalToNoiseRatio": 3.112385, "bySeverity": { "critical": { "recall": 0.5, "precision": 0.901961, "precisionScored": 51 }, "high": { "recall": 0.659091, "precision": 0.830399, "precisionScored": 1079 }, "low": { "recall": 0.483871, "precision": 0.752941, "precisionScored": 85 }, "medium": { "recall": 0.830303, "precision": 0.802846, "precisionScored": 492 }, "none": { "recall": null, "precision": 0.25, "precisionScored": 80 } }, "totalCost": 1498.15, "costPerReview": 2.560935, "latencyMsP50": 518544, "latencyMsP90": 2088625, "latencyMsMax": 3529837, "effectiveModel": { "provider": "xai", "modelName": "grok-4.6", "effort": "medium", "contextWindow": 500000 } }, "comparableAvg3": true, "baseModel": "Grok 4.6", "displayModel": "Grok 4.6 · medium" } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4.1 Flash · max", "provider": "DeepSeek", "metrics": { "score": { "value": 71.9851, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.7506, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 977.285, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 90.9722, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 83.1019, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 63.4916, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 71.4286, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 90.1515, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 84.2424, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 76.3441, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.2748, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 1818.966, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 2816.489, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "dsv4p1-max-control", "model": "DeepSeek V4.1 Flash · max", "provider": "DeepSeek", "effort": "max", "k": 3, "acc": 71.9851, "f1Score": 71.9851, "cost": 0.750621, "costPerReview": 0.750621, "lat": 977.285, "latencyMsP50": 977285, "recallAvgK": 83.1019, "precisionAvgK": 63.4916, "recallCritical": 71.4286, "recallHigh": 90.1515, "recallMedium": 84.2424, "recallLow": 76.3441, "recallMaxK": 90.9722, "snr": 1.27483, "latencyP90": 1818.966, "latencyMax": 2816.489, "totalCost": 439.11, "rankEligible": true, "benchmarkId": "52ccc297-28aa-58d1-914f-9b8e9ba00e16", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.719851, "recallAvgK": 0.831019, "recallMaxK": 0.909722, "precisionAvgK": 0.634916, "signalToNoiseRatio": 1.27483, "bySeverity": { "critical": { "recall": 0.714286, "precision": 0.945055, "precisionScored": 91 }, "high": { "recall": 0.901515, "precision": 0.758674, "precisionScored": 1297 }, "low": { "recall": 0.763441, "precision": 0.57315, "precisionScored": 581 }, "medium": { "recall": 0.842424, "precision": 0.725352, "precisionScored": 852 }, "none": { "recall": null, "precision": 0.18146, "precisionScored": 507 } }, "totalCost": 439.11, "costPerReview": 0.750621, "latencyMsP50": 977285, "latencyMsP90": 1818966, "latencyMsMax": 2816489, "effectiveModel": { "provider": "fireworks", "modelName": "deepseek-ai/DeepSeek-V4.1-Flash", "effort": "max", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "DeepSeek V4.1 Flash", "displayModel": "DeepSeek V4.1 Flash · max" } }, { "rank": 9, "ranked": true, "model": "Gemini 3.8 Flash · medium", "provider": "Google", "metrics": { "score": { "value": 70.7316, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 3.3842, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 663.127, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 70.8333, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 64.1204, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 78.8628, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 38.0952, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 65.1515, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 73.9394, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 56.9892, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 2.3374, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 1589.509, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 2430.274, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "gemini38-med-control", "model": "Gemini 3.8 Flash · medium", "provider": "Google", "effort": "medium", "k": 3, "acc": 70.7316, "f1Score": 70.7316, "cost": 3.384192, "costPerReview": 3.384192, "lat": 663.127, "latencyMsP50": 663127, "recallAvgK": 64.1204, "precisionAvgK": 78.8628, "recallCritical": 38.0952, "recallHigh": 65.1515, "recallMedium": 73.9394, "recallLow": 56.9892, "recallMaxK": 70.8333, "snr": 2.337449, "latencyP90": 1589.509, "latencyMax": 2430.274, "totalCost": 1979.75, "rankEligible": true, "benchmarkId": "99c74d2e-77c4-5df2-a3fe-d67a6b5acbfd", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.707316, "recallAvgK": 0.641204, "recallMaxK": 0.708333, "precisionAvgK": 0.788628, "signalToNoiseRatio": 2.337449, "bySeverity": { "critical": { "recall": 0.380952, "precision": 0.93617, "precisionScored": 47 }, "high": { "recall": 0.651515, "precision": 0.816826, "precisionScored": 939 }, "low": { "recall": 0.569892, "precision": 0.847458, "precisionScored": 118 }, "medium": { "recall": 0.739394, "precision": 0.828283, "precisionScored": 396 }, "none": { "recall": null, "precision": 0.313559, "precisionScored": 118 } }, "totalCost": 1979.75, "costPerReview": 3.384192, "latencyMsP50": 663127, "latencyMsP90": 1589509, "latencyMsMax": 2430274, "effectiveModel": { "provider": "vertexai", "modelName": "gemini-3.8-flash", "effort": "medium", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Gemini 3.8 Flash", "displayModel": "Gemini 3.8 Flash · medium" } }, { "rank": 10, "ranked": true, "model": "DeepSeek V4.1 Flash · low", "provider": "DeepSeek", "metrics": { "score": { "value": 70.721, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.384, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 489.42, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 86.8056, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 77.5463, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 65, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 69.0476, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 76.5152, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 81.8182, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 75.2688, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.3699, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 1009.848, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 2156.195, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "dsv4p1-low-control", "model": "DeepSeek V4.1 Flash · low", "provider": "DeepSeek", "effort": "low", "k": 3, "acc": 70.721, "f1Score": 70.721, "cost": 0.383983, "costPerReview": 0.383983, "lat": 489.42, "latencyMsP50": 489420, "recallAvgK": 77.5463, "precisionAvgK": 65, "recallCritical": 69.0476, "recallHigh": 76.5152, "recallMedium": 81.8182, "recallLow": 75.2688, "recallMaxK": 86.8056, "snr": 1.369948, "latencyP90": 1009.848, "latencyMax": 2156.195, "totalCost": 224.63, "rankEligible": true, "benchmarkId": "814e5ec4-8c99-5ac9-959d-35dc4362caec", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.70721, "recallAvgK": 0.775463, "recallMaxK": 0.868056, "precisionAvgK": 0.65, "signalToNoiseRatio": 1.369948, "bySeverity": { "critical": { "recall": 0.690476, "precision": 0.818182, "precisionScored": 55 }, "high": { "recall": 0.765152, "precision": 0.740958, "precisionScored": 1023 }, "low": { "recall": 0.752688, "precision": 0.610169, "precisionScored": 354 }, "medium": { "recall": 0.818182, "precision": 0.718412, "precisionScored": 554 }, "none": { "recall": null, "precision": 0.221088, "precisionScored": 294 } }, "totalCost": 224.63, "costPerReview": 0.383983, "latencyMsP50": 489420, "latencyMsP90": 1009848, "latencyMsMax": 2156195, "effectiveModel": { "provider": "fireworks", "modelName": "deepseek-ai/DeepSeek-V4.1-Flash", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "DeepSeek V4.1 Flash", "displayModel": "DeepSeek V4.1 Flash · low" } }, { "rank": 11, "ranked": true, "model": "DeepSeek V4.1 Flash · high", "provider": "DeepSeek", "metrics": { "score": { "value": 70.503, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.3762, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 460.21, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 85.4167, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 77.5463, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 64.6326, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 64.2857, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 79.5455, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 82.4242, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 72.043, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.3287, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 927.771, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 1659.534, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "dsv4p1-high-control", "model": "DeepSeek V4.1 Flash · high", "provider": "DeepSeek", "effort": "high", "k": 3, "acc": 70.503, "f1Score": 70.503, "cost": 0.376166, "costPerReview": 0.376166, "lat": 460.21, "latencyMsP50": 460210, "recallAvgK": 77.5463, "precisionAvgK": 64.6326, "recallCritical": 64.2857, "recallHigh": 79.5455, "recallMedium": 82.4242, "recallLow": 72.043, "recallMaxK": 85.4167, "snr": 1.328685, "latencyP90": 927.771, "latencyMax": 1659.534, "totalCost": 220.06, "rankEligible": true, "benchmarkId": "4b6f1bfc-9d2d-5124-bfa4-6f0295c67ad7", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.70503, "recallAvgK": 0.775463, "recallMaxK": 0.854167, "precisionAvgK": 0.646326, "signalToNoiseRatio": 1.328685, "bySeverity": { "critical": { "recall": 0.642857, "precision": 0.790323, "precisionScored": 62 }, "high": { "recall": 0.795455, "precision": 0.752964, "precisionScored": 1012 }, "low": { "recall": 0.72043, "precision": 0.601671, "precisionScored": 359 }, "medium": { "recall": 0.824242, "precision": 0.704861, "precisionScored": 576 }, "none": { "recall": null, "precision": 0.22327, "precisionScored": 318 } }, "totalCost": 220.06, "costPerReview": 0.376166, "latencyMsP50": 460210, "latencyMsP90": 927771, "latencyMsMax": 1659534, "effectiveModel": { "provider": "fireworks", "modelName": "deepseek-ai/DeepSeek-V4.1-Flash", "effort": "high", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "DeepSeek V4.1 Flash", "displayModel": "DeepSeek V4.1 Flash · high" } }, { "rank": 12, "ranked": true, "model": "Claude Opus 5 · high", "provider": "Anthropic", "metrics": { "score": { "value": 69.8731, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 6.5013, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 257.922, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 82.6389, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 76.6204, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 64.218, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 57.1429, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 74.2424, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 87.8788, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 68.8172, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.204, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 504.313, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 724.465, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "opus5-high-control", "model": "Claude Opus 5 · high", "provider": "Anthropic", "effort": "high", "k": 3, "acc": 69.8731, "f1Score": 69.8731, "cost": 6.501344, "costPerReview": 6.501344, "lat": 257.922, "latencyMsP50": 257922, "recallAvgK": 76.6204, "precisionAvgK": 64.218, "recallCritical": 57.1429, "recallHigh": 74.2424, "recallMedium": 87.8788, "recallLow": 68.8172, "recallMaxK": 82.6389, "snr": 1.203973, "latencyP90": 504.313, "latencyMax": 724.465, "totalCost": 3803.29, "rankEligible": true, "benchmarkId": "78cb4dc8-3927-5060-8d0c-14ae662c19ba", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.698731, "recallAvgK": 0.766204, "recallMaxK": 0.826389, "precisionAvgK": 0.64218, "signalToNoiseRatio": 1.203973, "bySeverity": { "critical": { "recall": 0.571429, "precision": 0.814286, "precisionScored": 70 }, "high": { "recall": 0.742424, "precision": 0.794136, "precisionScored": 1569 }, "low": { "recall": 0.688172, "precision": 0.572634, "precisionScored": 1067 }, "medium": { "recall": 0.878788, "precision": 0.703086, "precisionScored": 1199 }, "none": { "recall": null, "precision": 0.245223, "precisionScored": 628 } }, "totalCost": 3803.29, "costPerReview": 6.501344, "latencyMsP50": 257922, "latencyMsP90": 504313, "latencyMsMax": 724465, "effectiveModel": { "provider": "anthropic", "modelName": "claude-opus-5", "effort": "high", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Claude Opus 5", "displayModel": "Claude Opus 5 · high" } }, { "rank": 13, "ranked": true, "model": "GLM 5.3 · high", "provider": "Z.ai", "metrics": { "score": { "value": 69.4068, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 2.0174, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 328.949, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 77.0833, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 64.5833, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 75.0089, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 50, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 65.1515, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 72.1212, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 56.9892, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.9357, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 787.528, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 1860.395, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "glm53-high-control", "model": "GLM 5.3 · high", "provider": "Z.ai", "effort": "high", "k": 3, "acc": 69.4068, "f1Score": 69.4068, "cost": 2.017406, "costPerReview": 2.017406, "lat": 328.949, "latencyMsP50": 328949, "recallAvgK": 64.5833, "precisionAvgK": 75.0089, "recallCritical": 50, "recallHigh": 65.1515, "recallMedium": 72.1212, "recallLow": 56.9892, "recallMaxK": 77.0833, "snr": 1.935685, "latencyP90": 787.528, "latencyMax": 1860.395, "totalCost": 1180.18, "rankEligible": true, "benchmarkId": "3541ae6b-327d-535e-a954-f08bf733938e", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.694068, "recallAvgK": 0.645833, "recallMaxK": 0.770833, "precisionAvgK": 0.750089, "signalToNoiseRatio": 1.935685, "bySeverity": { "critical": { "recall": 0.5, "precision": 0.931507, "precisionScored": 73 }, "high": { "recall": 0.651515, "precision": 0.807528, "precisionScored": 1169 }, "low": { "recall": 0.569892, "precision": 0.665563, "precisionScored": 604 }, "medium": { "recall": 0.721212, "precision": 0.784337, "precisionScored": 830 }, "none": { "recall": null, "precision": 0.351724, "precisionScored": 145 } }, "totalCost": 1180.18, "costPerReview": 2.017406, "latencyMsP50": 328949, "latencyMsP90": 787528, "latencyMsMax": 1860395, "effectiveModel": { "provider": "fireworks", "modelName": "zhipuai/GLM-5.3", "effort": "high", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GLM 5.3", "displayModel": "GLM 5.3 · high" } }, { "rank": 14, "ranked": true, "model": "Kimi K3 · high", "provider": "Moonshot AI", "metrics": { "score": { "value": 68.5648, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 1.8952, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 260.902, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 71.5278, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 62.037, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 76.6279, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 47.619, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 62.1212, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 72.1212, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 50.5376, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.887, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 669.326, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 2541.027, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "k3-high-control", "model": "Kimi K3 · high", "provider": "Moonshot AI", "effort": "high", "k": 3, "acc": 68.5648, "f1Score": 68.5648, "cost": 1.895205, "costPerReview": 1.895205, "lat": 260.902, "latencyMsP50": 260902, "recallAvgK": 62.037, "precisionAvgK": 76.6279, "recallCritical": 47.619, "recallHigh": 62.1212, "recallMedium": 72.1212, "recallLow": 50.5376, "recallMaxK": 71.5278, "snr": 1.886957, "latencyP90": 669.326, "latencyMax": 2541.027, "totalCost": 1108.69, "rankEligible": true, "benchmarkId": "082ea61d-001a-5fd3-ab7a-9bbb4c8847d8", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.685648, "recallAvgK": 0.62037, "recallMaxK": 0.715278, "precisionAvgK": 0.766279, "signalToNoiseRatio": 1.886957, "bySeverity": { "critical": { "recall": 0.47619, "precision": 0.910256, "precisionScored": 78 }, "high": { "recall": 0.621212, "precision": 0.832278, "precisionScored": 948 }, "low": { "recall": 0.505376, "precision": 0.696607, "precisionScored": 501 }, "medium": { "recall": 0.721212, "precision": 0.805338, "precisionScored": 637 }, "none": { "recall": null, "precision": 0.354839, "precisionScored": 155 } }, "totalCost": 1108.69, "costPerReview": 1.895205, "latencyMsP50": 260902, "latencyMsP90": 669326, "latencyMsMax": 2541027, "effectiveModel": { "provider": "fireworks", "modelName": "moonshotai/Kimi-K3", "effort": "high", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Kimi K3", "displayModel": "Kimi K3 · high" } }, { "rank": 15, "ranked": true, "model": "GPT-5.6 Sol · low", "provider": "OpenAI", "metrics": { "score": { "value": 66.1098, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.8059, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 63.909, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 61.8056, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 55.0926, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 82.6347, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 39.2857, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 57.9545, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 65.4545, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 43.5484, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 3.1667, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 114.25, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 298.891, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "sol-low-control", "model": "GPT-5.6 Sol · low", "provider": "OpenAI", "effort": "low", "k": 3, "acc": 66.1098, "f1Score": 66.1098, "cost": 0.805913, "costPerReview": 0.805913, "lat": 63.909, "latencyMsP50": 63909, "recallAvgK": 55.0926, "precisionAvgK": 82.6347, "recallCritical": 39.2857, "recallHigh": 57.9545, "recallMedium": 65.4545, "recallLow": 43.5484, "recallMaxK": 61.8056, "snr": 3.166667, "latencyP90": 114.25, "latencyMax": 298.891, "totalCost": 471.46, "rankEligible": true, "benchmarkId": "1e0dc077-2b51-5853-a08a-3300116ae4ce", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.661098, "recallAvgK": 0.550926, "recallMaxK": 0.618056, "precisionAvgK": 0.826347, "signalToNoiseRatio": 3.166667, "bySeverity": { "critical": { "recall": 0.392857, "precision": 0.88, "precisionScored": 25 }, "high": { "recall": 0.579545, "precision": 0.82406, "precisionScored": 665 }, "low": { "recall": 0.435484, "precision": 0.884615, "precisionScored": 52 }, "medium": { "recall": 0.654545, "precision": 0.867036, "precisionScored": 361 }, "none": { "recall": null, "precision": 0.225, "precisionScored": 40 } }, "totalCost": 471.46, "costPerReview": 0.805913, "latencyMsP50": 63909, "latencyMsP90": 114250, "latencyMsMax": 298891, "effectiveModel": { "provider": "openai", "modelName": "gpt-5.6-sol", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GPT-5.6 Sol", "displayModel": "GPT-5.6 Sol · low" } }, { "rank": 16, "ranked": true, "model": "Claude Opus 5 · medium", "provider": "Anthropic", "metrics": { "score": { "value": 65.4963, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 3.496, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 131.671, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 76.3889, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 68.75, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 62.5366, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 47.619, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 70.4545, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 76.3636, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 62.3656, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.1815, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 310.143, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 858.378, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "opus5-med-control", "model": "Claude Opus 5 · medium", "provider": "Anthropic", "effort": "medium", "k": 3, "acc": 65.4963, "f1Score": 65.4963, "cost": 3.496005, "costPerReview": 3.496005, "lat": 131.671, "latencyMsP50": 131671, "recallAvgK": 68.75, "precisionAvgK": 62.5366, "recallCritical": 47.619, "recallHigh": 70.4545, "recallMedium": 76.3636, "recallLow": 62.3656, "recallMaxK": 76.3889, "snr": 1.181529, "latencyP90": 310.143, "latencyMax": 858.378, "totalCost": 2045.16, "rankEligible": true, "benchmarkId": "3e9b4fb2-1fcd-505c-9bce-90f7111fd25e", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.654963, "recallAvgK": 0.6875, "recallMaxK": 0.763889, "precisionAvgK": 0.625366, "signalToNoiseRatio": 1.181529, "bySeverity": { "critical": { "recall": 0.47619, "precision": 0.846154, "precisionScored": 52 }, "high": { "recall": 0.704545, "precision": 0.755887, "precisionScored": 1274 }, "low": { "recall": 0.623656, "precision": 0.559471, "precisionScored": 681 }, "medium": { "recall": 0.763636, "precision": 0.732708, "precisionScored": 853 }, "none": { "recall": null, "precision": 0.220217, "precisionScored": 554 } }, "totalCost": 2045.16, "costPerReview": 3.496005, "latencyMsP50": 131671, "latencyMsP90": 310143, "latencyMsMax": 858378, "effectiveModel": { "provider": "anthropic", "modelName": "claude-opus-5", "effort": "medium", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Claude Opus 5", "displayModel": "Claude Opus 5 · medium" } }, { "rank": 17, "ranked": true, "model": "Claude Opus 5 · low", "provider": "Anthropic", "metrics": { "score": { "value": 59.3759, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 1.4036, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 56.006, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 68.0556, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 57.4074, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 61.4843, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 40.4762, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 59.0909, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 64.2424, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 50.5376, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.116, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 137.597, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 282.155, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "opus5-low-control", "model": "Claude Opus 5 · low", "provider": "Anthropic", "effort": "low", "k": 3, "acc": 59.3759, "f1Score": 59.3759, "cost": 1.403637, "costPerReview": 1.403637, "lat": 56.006, "latencyMsP50": 56006, "recallAvgK": 57.4074, "precisionAvgK": 61.4843, "recallCritical": 40.4762, "recallHigh": 59.0909, "recallMedium": 64.2424, "recallLow": 50.5376, "recallMaxK": 68.0556, "snr": 1.115956, "latencyP90": 137.597, "latencyMax": 282.155, "totalCost": 821.13, "rankEligible": true, "benchmarkId": "e8feb853-cb76-5b43-afa4-ccb4d06477d9", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cd4560f98b6c769f336bd75e8783f983e4ce8f5ffdb534fcf05f797f08c30859", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.593759, "recallAvgK": 0.574074, "recallMaxK": 0.680556, "precisionAvgK": 0.614843, "signalToNoiseRatio": 1.115956, "bySeverity": { "critical": { "recall": 0.404762, "precision": 0.863636, "precisionScored": 44 }, "high": { "recall": 0.590909, "precision": 0.759672, "precisionScored": 853 }, "low": { "recall": 0.505376, "precision": 0.636364, "precisionScored": 297 }, "medium": { "recall": 0.642424, "precision": 0.676923, "precisionScored": 520 }, "none": { "recall": null, "precision": 0.19759, "precisionScored": 415 } }, "totalCost": 821.13, "costPerReview": 1.403637, "latencyMsP50": 56006, "latencyMsP90": 137597, "latencyMsMax": 282155, "effectiveModel": { "provider": "anthropic", "modelName": "claude-opus-5", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Claude Opus 5", "displayModel": "Claude Opus 5 · low" } }, { "rank": 18, "ranked": true, "model": "Grok 4.6 · low", "provider": "xAI", "metrics": { "score": { "value": 57.1025, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.298, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 54.939, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 59.7222, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 52.5463, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 62.5239, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 42.8571, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 51.5152, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 64.2424, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 37.6344, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.305, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 113.728, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 729.184, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "grok46-low-control", "model": "Grok 4.6 · low", "provider": "xAI", "effort": "low", "k": 3, "acc": 57.1025, "f1Score": 57.1025, "cost": 0.298021, "costPerReview": 0.298021, "lat": 54.939, "latencyMsP50": 54939, "recallAvgK": 52.5463, "precisionAvgK": 62.5239, "recallCritical": 42.8571, "recallHigh": 51.5152, "recallMedium": 64.2424, "recallLow": 37.6344, "recallMaxK": 59.7222, "snr": 1.304985, "latencyP90": 113.728, "latencyMax": 729.184, "totalCost": 174.34, "rankEligible": true, "benchmarkId": "19b1e774-2306-5d68-a71a-1476e94ffb01", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.571025, "recallAvgK": 0.525463, "recallMaxK": 0.597222, "precisionAvgK": 0.625239, "signalToNoiseRatio": 1.304985, "bySeverity": { "critical": { "recall": 0.428571, "precision": 0.733333, "precisionScored": 30 }, "high": { "recall": 0.515152, "precision": 0.718708, "precisionScored": 743 }, "low": { "recall": 0.376344, "precision": 0.583333, "precisionScored": 72 }, "medium": { "recall": 0.642424, "precision": 0.666667, "precisionScored": 486 }, "none": { "recall": null, "precision": 0.247899, "precisionScored": 238 } }, "totalCost": 174.34, "costPerReview": 0.298021, "latencyMsP50": 54939, "latencyMsP90": 113728, "latencyMsMax": 729184, "effectiveModel": { "provider": "xai", "modelName": "grok-4.6", "effort": "low", "contextWindow": 500000 } }, "comparableAvg3": true, "baseModel": "Grok 4.6", "displayModel": "Grok 4.6 · low" } }, { "rank": 19, "ranked": true, "model": "GLM 5.3 · low", "provider": "Z.ai", "metrics": { "score": { "value": 54.5613, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.408, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 82.365, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 61.8056, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 50, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 60.0383, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 52.381, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 50, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 57.5758, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 35.4839, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.1677, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 299.573, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 1268.588, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "glm53-low-control", "model": "GLM 5.3 · low", "provider": "Z.ai", "effort": "low", "k": 3, "acc": 54.5613, "f1Score": 54.5613, "cost": 0.408017, "costPerReview": 0.408017, "lat": 82.365, "latencyMsP50": 82365, "recallAvgK": 50, "precisionAvgK": 60.0383, "recallCritical": 52.381, "recallHigh": 50, "recallMedium": 57.5758, "recallLow": 35.4839, "recallMaxK": 61.8056, "snr": 1.167702, "latencyP90": 299.573, "latencyMax": 1268.588, "totalCost": 238.69, "rankEligible": true, "benchmarkId": "2895e1db-dd2a-5461-b20e-3ec66671926a", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.545613, "recallAvgK": 0.5, "recallMaxK": 0.618056, "precisionAvgK": 0.600383, "signalToNoiseRatio": 1.167702, "bySeverity": { "critical": { "recall": 0.52381, "precision": 0.833333, "precisionScored": 42 }, "high": { "recall": 0.5, "precision": 0.7067, "precisionScored": 791 }, "low": { "recall": 0.354839, "precision": 0.59375, "precisionScored": 352 }, "medium": { "recall": 0.575758, "precision": 0.674783, "precisionScored": 575 }, "none": { "recall": null, "precision": 0.189602, "precisionScored": 327 } }, "totalCost": 238.69, "costPerReview": 0.408017, "latencyMsP50": 82365, "latencyMsP90": 299573, "latencyMsMax": 1268588, "effectiveModel": { "provider": "fireworks", "modelName": "zhipuai/GLM-5.3", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "GLM 5.3", "displayModel": "GLM 5.3 · low" } }, { "rank": 20, "ranked": true, "model": "Kimi K3 · low", "provider": "Moonshot AI", "metrics": { "score": { "value": 53.5218, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.482, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 63.179, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 55.5556, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 45.1389, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 65.7284, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 38.0952, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 39.3939, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 55.1515, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 38.7097, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.3661, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 179.884, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 484.449, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "k3-low-control", "model": "Kimi K3 · low", "provider": "Moonshot AI", "effort": "low", "k": 3, "acc": 53.5218, "f1Score": 53.5218, "cost": 0.48202, "costPerReview": 0.48202, "lat": 63.179, "latencyMsP50": 63179, "recallAvgK": 45.1389, "precisionAvgK": 65.7284, "recallCritical": 38.0952, "recallHigh": 39.3939, "recallMedium": 55.1515, "recallLow": 38.7097, "recallMaxK": 55.5556, "snr": 1.366057, "latencyP90": 179.884, "latencyMax": 484.449, "totalCost": 281.98, "rankEligible": true, "benchmarkId": "d37f6c88-f0ff-54b2-ba8d-35f44a196366", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.535218, "recallAvgK": 0.451389, "recallMaxK": 0.555556, "precisionAvgK": 0.657284, "signalToNoiseRatio": 1.366057, "bySeverity": { "critical": { "recall": 0.380952, "precision": 0.9, "precisionScored": 40 }, "high": { "recall": 0.393939, "precision": 0.7411, "precisionScored": 618 }, "low": { "recall": 0.387097, "precision": 0.596059, "precisionScored": 203 }, "medium": { "recall": 0.551515, "precision": 0.736434, "precisionScored": 387 }, "none": { "recall": null, "precision": 0.196532, "precisionScored": 173 } }, "totalCost": 281.98, "costPerReview": 0.48202, "latencyMsP50": 63179, "latencyMsP90": 179884, "latencyMsMax": 484449, "effectiveModel": { "provider": "fireworks", "modelName": "moonshotai/Kimi-K3", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Kimi K3", "displayModel": "Kimi K3 · low" } }, { "rank": 21, "ranked": true, "model": "Gemini 3.8 Flash · low", "provider": "Google", "metrics": { "score": { "value": 51.3346, "unit": "percent", "label": "Score (avg@3)", "source_key": "f1Score" }, "cost_per_task": { "value": 0.9011, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 158.429, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "recallMaxK": { "value": 50.6944, "unit": "percent", "label": "Recall (any-of-k)", "source_key": "recallMaxK" }, "recallAvgK": { "value": 44.4444, "unit": "percent", "label": "Recall", "source_key": "recallAvgK" }, "precisionAvgK": { "value": 60.7532, "unit": "percent", "label": "Precision", "source_key": "precisionAvgK" }, "recallCritical": { "value": 16.6667, "unit": "percent", "label": "Recall (Critical severity)", "source_key": "recallCritical" }, "recallHigh": { "value": 48.4848, "unit": "percent", "label": "Recall (High severity)", "source_key": "recallHigh" }, "recallMedium": { "value": 53.9394, "unit": "percent", "label": "Recall (Medium severity)", "source_key": "recallMedium" }, "recallLow": { "value": 34.4086, "unit": "percent", "label": "Recall (Low severity)", "source_key": "recallLow" }, "snr": { "value": 1.1062, "unit": "valid findings per noise finding", "label": "Signal-to-noise ratio", "source_key": "snr" }, "latencyP90": { "value": 566.957, "unit": "seconds", "label": "Latency p90", "source_key": "latencyP90" }, "latencyMax": { "value": 1848.816, "unit": "seconds", "label": "Latency maximum", "source_key": "latencyMax" }, "k": { "value": 3, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "id": "gemini38-low-control", "model": "Gemini 3.8 Flash · low", "provider": "Google", "effort": "low", "k": 3, "acc": 51.3346, "f1Score": 51.3346, "cost": 0.901098, "costPerReview": 0.901098, "lat": 158.429, "latencyMsP50": 158429, "recallAvgK": 44.4444, "precisionAvgK": 60.7532, "recallCritical": 16.6667, "recallHigh": 48.4848, "recallMedium": 53.9394, "recallLow": 34.4086, "recallMaxK": 50.6944, "snr": 1.10625, "latencyP90": 566.957, "latencyMax": 1848.816, "totalCost": 527.14, "rankEligible": true, "benchmarkId": "2e48442d-1b30-51cd-b40c-41012b8e9168", "benchmarkApiVersion": "2026-09-11", "benchmarkDatasetId": "cf1c76e384da0db1180a88c48f437bb80559576b08a87e257eb44be5762e5c72", "benchmarkHarnessVersion": "4d954bee2fb72fe444118c86d5add75af41936170ba8db1ce797b2981ca161d4", "status": "completed", "sourceResults": { "k": 3, "f1score": 0.513346, "recallAvgK": 0.444444, "recallMaxK": 0.506944, "precisionAvgK": 0.607532, "signalToNoiseRatio": 1.10625, "bySeverity": { "critical": { "recall": 0.166667, "precision": 0.578947, "precisionScored": 57 }, "high": { "recall": 0.484848, "precision": 0.74902, "precisionScored": 510 }, "low": { "recall": 0.344086, "precision": 0.740741, "precisionScored": 54 }, "medium": { "recall": 0.539394, "precision": 0.712042, "precisionScored": 191 }, "none": { "recall": null, "precision": 0.111675, "precisionScored": 197 } }, "totalCost": 527.14, "costPerReview": 0.901098, "latencyMsP50": 158429, "latencyMsP90": 566957, "latencyMsMax": 1848816, "effectiveModel": { "provider": "vertexai", "modelName": "gemini-3.8-flash", "effort": "low", "contextWindow": 1000000 } }, "comparableAvg3": true, "baseModel": "Gemini 3.8 Flash", "displayModel": "Gemini 3.8 Flash · low" } } ] }, { "id": "site-reliability-engineering/traversal-orca-benchmark", "name": "ORCA Benchmark", "domain": "Software", "industry": "Site Reliability Engineering", "contributor": "Traversal", "description": "Root-cause analysis on live incidents, narrowing traces, logs, and metrics to the failing change.", "task_count": 324, "status": "published", "headline_metric": { "key": "hard", "label": "Score (Hard RCA, avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://www.traversal.com/blog/orca-bench-how-ready-are-language-model-agents-for-oncall", "source_title": "Read the public ORCA-bench methodology (not the source of these scores)", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-site-reliability-engineering%2Ftraversal-orca-benchmark", "methodology_summary": "ORCA-bench evaluates production-style root-cause analysis from ambiguous reports, telemetry, and source code. This board publishes results for nine models from a Fireworks-run evaluation of a 324-task subset. Hard RCA accuracy is the headline score; Medium RCA and incident hallucination remain separate native metrics. The public paper provides benchmark context, not these score rows.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 50.21, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 3.9989, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1143.73, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 65.99, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 3.75, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 1129568, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 1129568, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 0, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 38916, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 43.09, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "43.67–56.76%", "per_run_scores": [ 53.1646, 48.1013, 49.3671 ], "coverage": null, "note": "Passive-capture qualified; 106 and 144 superseded rows in Runs 2 and 3 pending cleanup, excluded from these metrics.", "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "hard": 50.21, "medium": 65.99, "halluc": 3.75, "runIds": [ 1, 2, 3 ], "hardCounts": [ 42, 38, 39 ], "mediumCounts": [ 64, 67, 63 ], "hallucCounts": [ 10, 11, 9 ], "cost": 3.9989, "lat": 1143.73, "inputTokens": 1129568, "cachedTokens": 0, "uncachedInput": 1129568, "recordedOutput": 38916, "toolCalls": 43.09, "caveat": "Passive-capture qualified; 106 and 144 superseded rows in Runs 2 and 3 pending cleanup, excluded from these metrics.", "acc": 50.21, "runs": [ 53.164556962025316, 48.10126582278481, 49.36708860759494 ], "mediumRuns": [ 65.3061224489796, 68.36734693877551, 64.28571428571429 ], "hallucRuns": [ 3.745318352059925, 4.119850187265918, 3.3707865168539324 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 43.66521728332062, "hi": 56.75672364494941, "halfWidth": 6.545753180814393, "runs": 3 }, "mediumCI": { "lo": 60.70971597328908, "hi": 71.26307314235719, "halfWidth": 5.27667858453405, "runs": 3 }, "hallucCI": { "lo": 2.814929695919483, "hi": 4.6757070082003676, "halfWidth": 0.930388656140442, "runs": 3 } } }, { "rank": 2, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 44.3038, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 5.3304, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2597.797, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 68.0272, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 6.9913, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 155908.0082, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 4901945.3827, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 4746037.3745, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 79325.3405, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 91.677, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "41.16–47.45%", "per_run_scores": [ 43.038, 44.3038, 45.5696 ], "coverage": null, "note": "Three reportable corrected-history runs; Run 1 carries passive-capture qualification.", "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "hard": 44.30379746835443, "medium": 68.02721088435374, "halluc": 6.991260923845194, "runIds": [ 1, 2, 3 ], "hardCounts": [ 34, 35, 36 ], "mediumCounts": [ 61, 69, 70 ], "hallucCounts": [ 15, 23, 18 ], "cost": 5.330422401234567, "lat": 2597.7972746913583, "inputTokens": 4901945.382716049, "cachedTokens": 4746037.374485597, "uncachedInput": 155908.0082304527, "recordedOutput": 79325.34053497942, "toolCalls": 91.67695473251028, "caveat": "Three reportable corrected-history runs; Run 1 carries passive-capture qualification.", "acc": 44.30379746835443, "runs": [ 43.037974683544306, 44.30379746835443, 45.56962025316456 ], "mediumRuns": [ 62.244897959183675, 70.40816326530613, 71.42857142857143 ], "hallucRuns": [ 5.617977528089888, 8.614232209737828, 6.741573033707865 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 41.159319352031666, "hi": 47.448275584677184, "halfWidth": 3.144478116322756, "runs": 3 }, "mediumCI": { "lo": 55.52316971431269, "hi": 80.53125205439481, "halfWidth": 12.504041170041061, "runs": 3 }, "hallucCI": { "lo": 3.2311399363295874, "hi": 10.7513819113608, "halfWidth": 3.7601209875156063, "runs": 3 } } }, { "rank": 3, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 32.07, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 2.4091, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1636.87, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 45.24, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 5.49, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 351519, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 2977180, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 2625660, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 65540, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 61.22, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "25.52–38.61%", "per_run_scores": [ 32.9114, 34.1772, 29.1139 ], "coverage": null, "note": "Runtime-qualified Run 1; accepted replacements and backend-evidence qualifications retained.", "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "hard": 32.07, "medium": 45.24, "halluc": 5.49, "runIds": [ 1, 2, 3 ], "hardCounts": [ 26, 27, 23 ], "mediumCounts": [ 47, 45, 41 ], "hallucCounts": [ 13, 15, 16 ], "cost": 2.4091, "lat": 1636.87, "inputTokens": 2977180, "cachedTokens": 2625660, "uncachedInput": 351519, "recordedOutput": 65540, "toolCalls": 61.22, "caveat": "Runtime-qualified Run 1; accepted replacements and backend-evidence qualifications retained.", "acc": 32.07, "runs": [ 32.91139240506329, 34.177215189873415, 29.11392405063291 ], "mediumRuns": [ 47.95918367346939, 45.91836734693877, 41.83673469387755 ], "hallucRuns": [ 4.868913857677903, 5.617977528089888, 5.992509363295881 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 25.521757367708815, "hi": 38.6132637293376, "halfWidth": 6.545753180814394, "runs": 3 }, "mediumCI": { "lo": 37.49404823030399, "hi": 52.98214224588649, "halfWidth": 7.744047007791252, "runs": 3 }, "hallucCI": { "lo": 4.071941435523953, "hi": 6.914325730518494, "halfWidth": 1.421192147497271, "runs": 3 } } }, { "rank": 4, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 31.22, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 1.24, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1345.23, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 52.04, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 10.24, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 76894, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 1666781, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 1589887, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 35525, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 73.716, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "15.71–46.73%", "per_run_scores": [ 34.1772, 24.0506, 35.443 ], "coverage": null, "note": "Three reportable corrected-history runs; Run 1 carries passive-capture qualification. Retained service-evidence qualifications.", "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "hard": 31.22, "medium": 52.04, "halluc": 10.24, "runIds": [ 1, 2, 3 ], "hardCounts": [ 27, 19, 28 ], "mediumCounts": [ 48, 49, 56 ], "hallucCounts": [ 30, 29, 23 ], "cost": 1.24, "lat": 1345.23, "inputTokens": 1666781, "cachedTokens": 1589887, "uncachedInput": 76894, "recordedOutput": 35525, "toolCalls": 73.71604938, "caveat": "Three reportable corrected-history runs; Run 1 carries passive-capture qualification. Retained service-evidence qualifications.", "acc": 31.22, "runs": [ 34.177215189873415, 24.050632911392405, 35.44303797468354 ], "mediumRuns": [ 48.97959183673469, 50, 57.142857142857146 ], "hallucRuns": [ 11.235955056179776, 10.861423220973784, 8.614232209737828 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 15.712286481046121, "hi": 46.734970902920125, "halfWidth": 15.511342210937002, "runs": 3 }, "mediumCI": { "lo": 40.99172933878717, "hi": 63.08990331427405, "halfWidth": 11.049086987743443, "runs": 3 }, "hallucCI": { "lo": 6.71480889283456, "hi": 13.759598098426364, "halfWidth": 3.5223946027959023, "runs": 3 } } }, { "rank": 5, "ranked": true, "model": "GLM-5.3", "provider": "Z.ai", "metrics": { "score": { "value": 29.96, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 2.3116, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 2418.769, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 56.8, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 6.37, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 239623.3405, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 5534843.4671, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 5295220.1265, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 136215.6481, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 204.3909, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "22.04–37.87%", "per_run_scores": [ 30.3797, 32.9114, 26.5823 ], "coverage": null, "note": "Run 1 passive-capture qualified; accepted recoveries and unresolved backend-evidence limits retained.", "fireworks_model_url": null, "raw": { "model": "GLM-5.3", "provider": "Z.ai", "hard": 29.96, "medium": 56.8, "halluc": 6.37, "runIds": [ 1, 2, 3 ], "hardCounts": [ 24, 26, 21 ], "mediumCounts": [ 56, 56, 55 ], "hallucCounts": [ 13, 16, 22 ], "cost": 2.3115676666255145, "lat": 2418.76933127572, "inputTokens": 5534843.467078189, "cachedTokens": 5295220.12654321, "uncachedInput": 239623.34053497945, "recordedOutput": 136215.64814814815, "toolCalls": 204.39094650205763, "caveat": "Run 1 passive-capture qualified; accepted recoveries and unresolved backend-evidence limits retained.", "acc": 29.96, "runs": [ 30.379746835443036, 32.91139240506329, 26.582278481012658 ], "mediumRuns": [ 57.142857142857146, 57.142857142857146, 56.12244897959184 ], "hallucRuns": [ 4.868913857677903, 5.992509363295881, 8.239700374531836 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 22.044376185983378, "hi": 37.87123562836261, "halfWidth": 7.913429721189618, "runs": 3 }, "mediumCI": { "lo": 55.339233765306126, "hi": 58.26620841156464, "halfWidth": 1.4634873231292556, "runs": 3 }, "hallucCI": { "lo": 2.1034647560100597, "hi": 10.630617640993687, "halfWidth": 4.2635764424918134, "runs": 3 } } }, { "rank": 6, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 29.54, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 1.4031, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1139.548, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 42.86, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 8.24, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 639980.0597, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 9160068.2459, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 8520088.1862, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 75756.3313, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 101.3477, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "15.36–43.72%", "per_run_scores": [ 24.0506, 35.443, 29.1139 ], "coverage": null, "note": "Runtime-qualified Run 1; five superseded Run 2 rows pending cleanup, excluded from these metrics.", "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "hard": 29.54, "medium": 42.86, "halluc": 8.24, "runIds": [ 1, 2, 3 ], "hardCounts": [ 19, 28, 23 ], "mediumCounts": [ 43, 43, 40 ], "hallucCounts": [ 23, 16, 27 ], "cost": 1.403077901003086, "lat": 1139.548021604938, "inputTokens": 9160068.245884774, "cachedTokens": 8520088.186213993, "uncachedInput": 639980.059670782, "recordedOutput": 75756.33127572016, "toolCalls": 101.3477366255144, "caveat": "Runtime-qualified Run 1; five superseded Run 2 rows pending cleanup, excluded from these metrics.", "acc": 29.54, "runs": [ 24.050632911392405, 35.44303797468354, 29.11392405063291 ], "mediumRuns": [ 43.87755102040816, 43.87755102040816, 40.816326530612244 ], "hallucRuns": [ 8.614232209737828, 5.992509363295881, 10.112359550561798 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 15.356627810117562, "hi": 43.71510214768834, "halfWidth": 14.179237168785392, "runs": 3 }, "mediumCI": { "lo": 38.466680887755096, "hi": 47.24760482653061, "halfWidth": 4.390461969387756, "runs": 3 }, "hallucCI": { "lo": 3.0595155712917714, "hi": 13.4198851777719, "halfWidth": 5.180184803240064, "runs": 3 } } }, { "rank": 6, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 29.54, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 1.965, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1435.82, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 46.94, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 6.62, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 72036, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 1112523, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 1040486, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 59434, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 110.46, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "24.73–34.34%", "per_run_scores": [ 29.1139, 27.8481, 31.6456 ], "coverage": null, "note": "Runtime-qualified Run 1; retained request-attribution limits. Superseded attempts excluded.", "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "hard": 29.54, "medium": 46.94, "halluc": 6.62, "runIds": [ 1, 2, 3 ], "hardCounts": [ 23, 22, 25 ], "mediumCounts": [ 48, 41, 49 ], "hallucCounts": [ 15, 19, 19 ], "cost": 1.965, "lat": 1435.82, "inputTokens": 1112523, "cachedTokens": 1040486, "uncachedInput": 72036, "recordedOutput": 59434, "toolCalls": 110.46, "caveat": "Runtime-qualified Run 1; retained request-attribution limits. Superseded attempts excluded.", "acc": 29.54, "runs": [ 29.11392405063291, 27.848101265822784, 31.645569620253166 ], "mediumRuns": [ 48.97959183673469, 41.83673469387755, 50 ], "hallucRuns": [ 5.617977528089888, 7.116104868913857, 7.116104868913857 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 24.732595315842552, "hi": 34.339134641963355, "halfWidth": 4.803269663060401, "runs": 3 }, "mediumCI": { "lo": 35.889688522460645, "hi": 57.9878624979475, "halfWidth": 11.049086987743433, "runs": 3 }, "hallucCI": { "lo": 4.46808852434457, "hi": 8.765369652933831, "halfWidth": 2.148640564294631, "runs": 3 } } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 24.47, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 0.5578, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 1128.49, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 37.41, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 9.74, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 128943, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 3948541, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 3819598, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 55436, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 119.49, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "17.93–31.02%", "per_run_scores": [ 21.519, 26.5823, 25.3165 ], "coverage": null, "note": "Corrected Run 1 accepted with passive-capture qualification; retained service-evidence limits.", "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "hard": 24.47, "medium": 37.41, "halluc": 9.74, "runIds": [ 1, 2, 3 ], "hardCounts": [ 17, 21, 20 ], "mediumCounts": [ 39, 39, 32 ], "hallucCounts": [ 26, 26, 26 ], "cost": 0.5578, "lat": 1128.49, "inputTokens": 3948541, "cachedTokens": 3819598, "uncachedInput": 128943, "recordedOutput": 55436, "toolCalls": 119.49, "caveat": "Corrected Run 1 accepted with passive-capture qualification; retained service-evidence limits.", "acc": 24.47, "runs": [ 21.518987341772153, 26.582278481012658, 25.31645569620253 ], "mediumRuns": [ 39.795918367346935, 39.795918367346935, 32.6530612244898 ], "hallucRuns": [ 9.737827715355806, 9.737827715355806, 9.737827715355806 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 17.926820658848055, "hi": 31.018327020476843, "halfWidth": 6.545753180814394, "runs": 3 }, "mediumCI": { "lo": 27.170554724489797, "hi": 47.659377248299315, "halfWidth": 10.244411261904757, "runs": 3 }, "hallucCI": { "lo": 9.737827715355806, "hi": 9.737827715355806, "halfWidth": 0, "runs": 3 } } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 20.25, "unit": "percent", "label": "Score (Hard RCA, avg@3)", "source_key": "hard" }, "cost_per_task": { "value": 0.1233, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 915.24, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "medium": { "value": 41.16, "unit": "percent", "label": "Score (Medium RCA, avg@3)", "source_key": "medium" }, "halluc": { "value": 9.99, "unit": "percent", "label": "Hallucination (incidents)", "source_key": "halluc" }, "uncachedInput": { "value": 160272, "unit": "tokens", "label": "Uncached input tokens / task", "source_key": "uncachedInput" }, "inputTokens": { "value": 5207222, "unit": "tokens", "label": "Input tokens (total)", "source_key": "inputTokens" }, "cachedTokens": { "value": 5046950, "unit": "tokens", "label": "Cached tokens (total)", "source_key": "cachedTokens" }, "recordedOutput": { "value": 79906, "unit": "tokens", "label": "Output tokens (total)", "source_key": "recordedOutput" }, "runsInMean": { "value": 3, "unit": "count", "label": "Reportable runs", "source_key": "runsInMean" }, "verdictsPerRun": { "value": 324, "unit": "count", "label": "Canonical verdicts per run", "source_key": "verdictsPerRun" }, "toolCalls": { "value": 120.88, "unit": "count", "label": "Tool calls / task", "source_key": "toolCalls" } }, "confidence_interval_95": "17.11–23.40%", "per_run_scores": [ 20.2532, 18.9873, 21.519 ], "coverage": null, "note": "Passive-capture qualified; one superseded Run 2 API row pending cleanup, excluded from these metrics.", "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "hard": 20.25, "medium": 41.16, "halluc": 9.99, "runIds": [ 1, 2, 3 ], "hardCounts": [ 16, 15, 17 ], "mediumCounts": [ 40, 40, 41 ], "hallucCounts": [ 25, 26, 29 ], "cost": 0.1233, "lat": 915.24, "inputTokens": 5207222, "cachedTokens": 5046950, "uncachedInput": 160272, "recordedOutput": 79906, "toolCalls": 120.88, "caveat": "Passive-capture qualified; one superseded Run 2 API row pending cleanup, excluded from these metrics.", "acc": 20.25, "runs": [ 20.253164556962027, 18.9873417721519, 21.518987341772153 ], "mediumRuns": [ 40.816326530612244, 40.816326530612244, 41.83673469387755 ], "hallucRuns": [ 9.363295880149813, 9.737827715355806, 10.861423220973784 ], "runsInMean": 3, "verdictsPerRun": 324, "ci": { "lo": 17.10868644063927, "hi": 23.397642673284782, "halfWidth": 3.144478116322756, "runs": 3 }, "mediumCI": { "lo": 39.692975261904756, "hi": 42.61994990816327, "halfWidth": 1.4634873231292556, "runs": 3 }, "hallucCI": { "lo": 8.05075717371659, "hi": 11.92427403726968, "halfWidth": 1.936758431776544, "runs": 3 } } } ] }, { "id": "healthcare/bedside-bench", "name": "Bedside Bench", "domain": "Healthcare", "industry": "Healthcare", "contributor": "Doximity", "description": "Five hundred physician-authored clinical cases spanning ten suites of medical reasoning and safety, from differential diagnosis and workup through escalation and medication safety. Each case is graded by an LLM judge against criteria the authoring physicians wrote. The headline score is the macro-average of the ten suite means, so no single suite can carry a model.", "task_count": 500, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@3)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://www.doximity.com", "source_title": "Open Doximity", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-healthcare%2Fbedside-bench", "methodology_summary": "Doximity’s physician-validated clinical question-answering benchmark covers factual questions, calculations, false-premise queries, and free-text management plans.", "board_note": null, "result_count": 10, "results": [ { "rank": 1, "ranked": true, "model": "Doximity Ask V7", "provider": "Doximity", "metrics": { "score": { "value": 95.4, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": null, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": null, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": null, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": null, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": "Doximity-reported macro mean · not rerun by Fireworks · board only", "fireworks_model_url": null, "raw": { "model": "Doximity Ask V7", "provider": "Doximity", "acc": 95.4, "boardOnly": true, "ci": null, "runScores": null, "runs": null, "cost": null, "lat": null, "coverage": null, "prompt": null, "cached": null, "completion": null, "note": "Doximity-reported macro mean · not rerun by Fireworks · board only", "se": null } }, { "rank": 2, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 94.6, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.1188, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 64.6, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 262.7, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 4698.7, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "92.7–96.4%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Official API · max reasoning · top-p omitted", "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 94.6, "ci": "92.7–96.4%", "runScores": "95.3 · 94.5 · 94.1", "runs": 3, "cost": 0.1188, "lat": 64.6, "coverage": "1,500 / 1,500", "prompt": 262.7, "cached": 0, "completion": 4698.7, "note": "Official API · max reasoning · top-p omitted", "se": null } }, { "rank": 3, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 93.4, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.106, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 53.6, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 161.2, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2086.9, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "91.8–94.9%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Official API · xhigh reasoning · top-p omitted", "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 93.4, "ci": "91.8–94.9%", "runScores": "93.4 · 93.2 · 93.6", "runs": 3, "cost": 0.106, "lat": 53.6, "coverage": "1,500 / 1,500", "prompt": 161.2, "cached": 0, "completion": 2086.9, "note": "Official API · xhigh reasoning · top-p omitted", "se": null } }, { "rank": 4, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 91.1, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0501, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 42, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 161.2, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2471.2, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "88.4–93.1%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Official API · xhigh reasoning · also used as judge", "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 91.1, "ci": "88.4–93.1%", "runScores": "91.5 · 91.6 · 90.3", "runs": 3, "cost": 0.0501, "lat": 42, "coverage": "1,500 / 1,500", "prompt": 161.2, "cached": 0, "completion": 2471.2, "note": "Official API · xhigh reasoning · also used as judge", "se": null } }, { "rank": 5, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 89.7, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0529, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 82.9, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 250.2, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 2.3, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 3479, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "87.3–91.7%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Fireworks API · max reasoning", "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 89.7, "ci": "87.3–91.7%", "runScores": "90.0 · 89.7 · 89.2", "runs": 3, "cost": 0.0529, "lat": 82.9, "coverage": "1,500 / 1,500", "prompt": 250.2, "cached": 2.3, "completion": 3479, "note": "Fireworks API · max reasoning", "se": null } }, { "rank": 6, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 86.9, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.021, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 94.7, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 106.9, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 59.3, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 4728.1, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "84.3–89.2%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Fireworks API · max reasoning", "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "acc": 86.9, "ci": "84.3–89.2%", "runScores": "87.0 · 87.2 · 86.4", "runs": 3, "cost": 0.021, "lat": 94.7, "coverage": "1,500 / 1,500", "prompt": 106.9, "cached": 59.3, "completion": 4728.1, "note": "Fireworks API · max reasoning", "se": null } }, { "rank": 7, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 86.5, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0092, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 10, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 155.4, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2412, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "84.2–88.5%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Official API · high thinking level", "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 86.5, "ci": "84.2–88.5%", "runScores": "86.2 · 86.5 · 86.7", "runs": 3, "cost": 0.0092, "lat": 10, "coverage": "1,500 / 1,500", "prompt": 155.4, "cached": 0, "completion": 2412, "note": "Official API · high thinking level", "se": null } }, { "rank": 8, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 83.3, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0119, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 38.9, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 257.7, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 533.6, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 1855.7, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "80.7–85.6%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Official API · xhigh reasoning", "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 83.3, "ci": "80.7–85.6%", "runScores": "83.6 · 83.5 · 82.8", "runs": 3, "cost": 0.0119, "lat": 38.9, "coverage": "1,500 / 1,500", "prompt": 257.7, "cached": 533.6, "completion": 1855.7, "note": "Official API · xhigh reasoning", "se": null } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 82.9, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0046, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 102, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 157.9, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 20.6, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 6923, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "80.0–85.8%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Fireworks API · max reasoning", "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "acc": 82.9, "ci": "80.0–85.8%", "runScores": "82.8 · 83.5 · 82.4", "runs": 3, "cost": 0.0046, "lat": 102, "coverage": "1,500 / 1,500", "prompt": 157.9, "cached": 20.6, "completion": 6923, "note": "Fireworks API · max reasoning", "se": null } }, { "rank": 10, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 80.6, "unit": "percent", "label": "Score (avg@3)", "source_key": "acc" }, "cost_per_task": { "value": 0.0157, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 62.9, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 243.3, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 1.2, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 3878, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "77.8–83.3%", "per_run_scores": null, "coverage": "1,500 / 1,500", "note": "Fireworks API · max reasoning", "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "acc": 80.6, "ci": "77.8–83.3%", "runScores": "80.5 · 81.1 · 80.1", "runs": 3, "cost": 0.0157, "lat": 62.9, "coverage": "1,500 / 1,500", "prompt": 243.3, "cached": 1.2, "completion": 3878, "note": "Fireworks API · max reasoning", "se": null } } ] }, { "id": "healthcare/apex-1-general-practitioner", "name": "APEX-1: General Practitioner", "domain": "Healthcare", "industry": "Healthcare", "contributor": "Mercor", "description": "Primary-care consults graded by practicing GPs on diagnosis, workup, and safe escalation. Each case is scored against the plan a GP would defend to a colleague rather than against one reference answer.", "task_count": 100, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@4)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "provider", "label": "Provider", "statement": "The original provider — the task set or harness stays with the benchmark owner." }, "source_url": "https://www.mercor.com", "source_title": "Open Mercor", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-healthcare%2Fapex-1-general-practitioner", "methodology_summary": "APEX-1 evaluates open-ended General Practitioner deliverables authored by practicing physicians and graded criterion by criterion. The Specialized Intelligence Index publishes the supplied four-attempt mean rubric scores and reported 95% confidence intervals.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 72.12, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 1.971, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 9.25, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 4.87, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.4928, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 52484, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "67.85–76.39%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "servedBy": "Anthropic", "modelIdentifier": "claude-opus-5", "reasoning": "max", "acc": 72.12, "ci": { "lo": 67.85, "hi": 76.39, "halfWidth": 4.27 }, "pass1": 9.25, "pass1CiHalfWidth": 4.87, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 1.971, "costPerAttempt": 0.4928, "avgTokens": 52484, "k": 4, "priceBasis": "anthropic/claude-opus-5", "batchId": "batch_5684157a97a64bf89872959b3da85f38" } }, { "rank": 2, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 68.36, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.204, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 6.75, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 4.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.0509, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 33634, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "64.77–71.95%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "servedBy": "Google Vertex", "modelIdentifier": "gemini-3.8-flash", "reasoning": "high", "acc": 68.36, "ci": { "lo": 64.77, "hi": 71.95, "halfWidth": 3.59 }, "pass1": 6.75, "pass1CiHalfWidth": 4.25, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.204, "costPerAttempt": 0.0509, "avgTokens": 33634, "k": 4, "priceBasis": "vertex_ai/gemini-3.8-flash", "batchId": "batch_c8243ea8b4b849c6ae53fdf164bb53dc" } }, { "rank": 3, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 64.35, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.692, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 5.25, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 4, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.173, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 30950, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "60.46–68.24%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/kimi-k3", "reasoning": "max", "acc": 64.35, "ci": { "lo": 60.46, "hi": 68.24, "halfWidth": 3.89 }, "pass1": 5.25, "pass1CiHalfWidth": 4, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.692, "costPerAttempt": 0.173, "avgTokens": 30950, "k": 4, "priceBasis": "fireworks_ai/.../kimi-k3", "batchId": "batch_b671f02eb31f4b068094ad474e089e25" } }, { "rank": 4, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 63.75, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 5.725, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 3.75, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 3.5, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 1.4312, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 190070, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "59.82–67.68%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "servedBy": "OpenAI", "modelIdentifier": "responses/gpt-5.6-sol", "reasoning": "max", "acc": 63.75, "ci": { "lo": 59.82, "hi": 67.68, "halfWidth": 3.93 }, "pass1": 3.75, "pass1CiHalfWidth": 3.5, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 5.725, "costPerAttempt": 1.4312, "avgTokens": 190070, "k": 4, "priceBasis": "openai/gpt-5.6-sol", "batchId": "batch_e367ed890f904c51817b25ac31a04bb9" } }, { "rank": 5, "ranked": true, "model": "GLM 5.3", "provider": "Z.ai", "metrics": { "score": { "value": 63.2, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.445, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 5.5, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 3.88, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.1121, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 397, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 41779, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "58.85–67.55%", "per_run_scores": null, "coverage": "397 / 400 (99.25%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GLM 5.3", "provider": "Z.ai", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/glm-5p3", "reasoning": "max", "acc": 63.2, "ci": { "lo": 58.85, "hi": 67.55, "halfWidth": 4.35 }, "pass1": 5.5, "pass1CiHalfWidth": 3.88, "attemptsScored": 397, "attemptsExpected": 400, "coverage": "397 / 400 (99.25%)", "cost": 0.445, "costPerAttempt": 0.1121, "avgTokens": 41779, "k": 4, "priceBasis": "fireworks_ai/.../glm-5p3", "batchId": "batch_617638bda24f47b9a8eef4889ce474f6" } }, { "rank": 6, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 61.03, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.709, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 6, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 4.38, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.1773, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 25006, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "56.77–65.29%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "servedBy": "OpenAI", "modelIdentifier": "responses/gpt-6-astra", "reasoning": "max", "acc": 61.03, "ci": { "lo": 56.77, "hi": 65.29, "halfWidth": 4.26 }, "pass1": 6, "pass1CiHalfWidth": 4.38, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.709, "costPerAttempt": 0.1773, "avgTokens": 25006, "k": 4, "priceBasis": "openai/gpt-6-astra", "batchId": "batch_7e68d7176b004039b74d8a4026a0f6dd" } }, { "rank": 7, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 59.38, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.235, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 3.5, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 2.88, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.0588, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 27644, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "55.14–63.62%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "servedBy": "xAI", "modelIdentifier": "grok-4.6", "reasoning": "xhigh", "acc": 59.38, "ci": { "lo": 55.14, "hi": 63.62, "halfWidth": 4.24 }, "pass1": 3.5, "pass1CiHalfWidth": 2.88, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.235, "costPerAttempt": 0.0588, "avgTokens": 27644, "k": 4, "priceBasis": "xai/grok-4.6", "batchId": "batch_ee71b46949d14f018f41846f35955152" } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 56.53, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.316, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 4.25, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 3.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.079, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 35442, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "52.10–60.96%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4-pro-0813", "reasoning": "max", "acc": 56.53, "ci": { "lo": 52.1, "hi": 60.96, "halfWidth": 4.43 }, "pass1": 4.25, "pass1CiHalfWidth": 3.25, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.316, "costPerAttempt": 0.079, "avgTokens": 35442, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4-pro-0813", "batchId": "batch_db354998a14d435ab17c57aebcf8d71a" } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 56.18, "unit": "percent", "label": "Score (avg@4)", "source_key": "acc" }, "cost_per_task": { "value": 0.056, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "pass1": { "value": 4, "unit": "percent", "label": "Pass@1", "source_key": "pass1" }, "pass1CiHalfWidth": { "value": 3.25, "unit": "percentage points", "label": "Pass@1 95% CI (+/-)", "source_key": "pass1CiHalfWidth" }, "costPerAttempt": { "value": 0.014, "unit": "USD", "label": "Cost / attempt", "source_key": "costPerAttempt" }, "k": { "value": 4, "unit": "count", "label": "Attempt budget (k)", "source_key": "k" }, "attemptsScored": { "value": 400, "unit": "count", "label": "Attempts scored", "source_key": "attemptsScored" }, "attemptsExpected": { "value": 400, "unit": "count", "label": "Attempts expected", "source_key": "attemptsExpected" }, "avgTokens": { "value": 36603, "unit": "tokens", "label": "Average tokens / attempt", "source_key": "avgTokens" } }, "confidence_interval_95": "51.71–60.65%", "per_run_scores": null, "coverage": "400 / 400 (100%)", "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "servedBy": "Fireworks", "modelIdentifier": "accounts/fireworks/models/deepseek-v4p1-flash", "reasoning": "max", "acc": 56.18, "ci": { "lo": 51.71, "hi": 60.65, "halfWidth": 4.47 }, "pass1": 4, "pass1CiHalfWidth": 3.25, "attemptsScored": 400, "attemptsExpected": 400, "coverage": "400 / 400 (100%)", "cost": 0.056, "costPerAttempt": 0.014, "avgTokens": 36603, "k": 4, "priceBasis": "fireworks_ai/.../deepseek-v4p1-flash", "batchId": "batch_a40c124259db4d18b70731fadc7ab97d" } } ] }, { "id": "healthcare/healthbench-professional", "name": "HealthBench Professional", "domain": "Healthcare", "industry": "Healthcare", "contributor": null, "description": "The professional-level slice of OpenAI's HealthBench: 525 clinical consults answered in free text and graded against physician-written rubrics. Scoring is length-adjusted, so restating the question earns nothing, and the headline is the mean weighted rubric score across every complete run.", "task_count": 525, "status": "published", "headline_metric": { "key": "acc", "label": "Score (avg@8)", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "public", "label": "Public", "statement": "The public — the task set and harness are openly available, so anyone can re-run this benchmark." }, "source_url": null, "source_title": null, "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-healthcare%2Fhealthbench-professional", "methodology_summary": "Physician-authored evaluation of accurate, useful, and safe responses to difficult professional clinical tasks.", "board_note": null, "result_count": 9, "results": [ { "rank": 1, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 64.43, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0533, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 26.5, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 560.3771, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 953.5282, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "63.95–64.69%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 64.43, "ci": "63.95–64.69%", "cost": 0.05328018015873016, "lat": 26.5, "prompt": 560.3771428571429, "cached": 0, "completion": 953.5281746031746, "se": null } }, { "rank": 2, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 58.83, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0727, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 56.8, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 560.3771, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 3521.653, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "54.90–62.76%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 58.83004008309629, "ci": "54.90–62.76%", "cost": 0.07267456888888889, "lat": 56.8, "prompt": 560.3771428571429, "cached": 0, "completion": 3521.653015873016, "se": null } }, { "rank": 3, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 55.4807, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.1399, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 76.1, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 893.8095, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 5418.6744, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "51.67–59.29%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 55.48073669137791, "ci": "51.67–59.29%", "cost": 0.1399359087301587, "lat": 76.1, "prompt": 893.8095238095239, "cached": 0, "completion": 5418.674444444445, "se": null } }, { "rank": 4, "ranked": true, "model": "Grok 4.6", "provider": "xAI", "metrics": { "score": { "value": 51.661, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0153, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 44.6, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 1183.4564, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2157.3525, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "47.63–55.69%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Grok 4.6", "provider": "xAI", "acc": 51.66096031464708, "ci": "47.63–55.69%", "cost": 0.01531102809523809, "lat": 44.6, "prompt": 1183.456428571428, "cached": 0, "completion": 2157.35253968254, "se": null } }, { "rank": 5, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 49.8163, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0083, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 8.6, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 603.3352, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 2095.4964, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "45.50–54.13%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 49.81629934392793, "ci": "45.50–54.13%", "cost": 0.008310613035714285, "lat": 8.6, "prompt": 603.3352380952381, "cached": 0, "completion": 2095.496428571429, "se": null } }, { "rank": 6, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 48.8752, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.053, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 67.3, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 672.1314, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 3398.9333, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "44.59–53.16%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 48.87517724174351, "ci": "44.59–53.16%", "cost": 0.05300039309523809, "lat": 67.3, "prompt": 672.1314285714286, "cached": 0, "completion": 3398.933253968254, "se": null } }, { "rank": 7, "ranked": true, "model": "GLM-5.3", "provider": "Z.ai", "metrics": { "score": { "value": 48.7264, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0311, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 97.4, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 582.1752, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 6885.3539, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "44.82–52.63%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GLM-5.3", "provider": "Z.ai", "acc": 48.72636284952446, "ci": "44.82–52.63%", "cost": 0.03111060261904763, "lat": 97.4, "prompt": 582.1752380952381, "cached": 0, "completion": 6885.353928571429, "se": null } }, { "rank": 8, "ranked": true, "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "metrics": { "score": { "value": 47.76, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.005, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 580.4057, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 7391.2517, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "47.58–48.10%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "acc": 47.76, "ci": "47.58–48.10%", "cost": 0.0050059153571428575, "lat": null, "prompt": 580.4057142857143, "cached": 0, "completion": 7391.251666666667, "se": null } }, { "rank": 9, "ranked": true, "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "metrics": { "score": { "value": 38.9052, "unit": "percent", "label": "Score (avg@8)", "source_key": "acc" }, "cost_per_task": { "value": 0.0202, "unit": "USD", "label": "Cost / task", "source_key": "cost" }, "duration_per_task": { "value": 63.7, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "se": { "value": null, "unit": "percentage points", "label": "Standard error", "source_key": "se" }, "prompt": { "value": 646.4057, "unit": "tokens", "label": "Prompt tokens / task", "source_key": "prompt" }, "cached": { "value": 0, "unit": "tokens", "label": "Cached tokens / task", "source_key": "cached" }, "completion": { "value": 4885.876, "unit": "tokens", "label": "Completion tokens / task", "source_key": "completion" } }, "confidence_interval_95": "34.15–43.66%", "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "acc": 38.90520475078049, "ci": "34.15–43.66%", "cost": 0.02020132462857143, "lat": 63.7, "prompt": 646.4057142857143, "cached": 0, "completion": 4885.876031746032, "se": null } } ] }, { "id": "slide-generation/genspark-slides-benchmark", "name": "Genspark Slides Benchmark", "domain": "Productivity", "industry": "Slide Generation", "contributor": "Genspark", "description": "Eleven models generate finished decks from 200 real production tasks in the Genspark Slides agent harness. Genspark's internal grader reports an aggregate score plus a quality composite, task completion, content quality, visual design, process quality, a layout-defect penalty, the Tier A deck share, and USD per deck.", "task_count": 200, "status": "published", "headline_metric": { "key": "acc", "label": "Score", "unit": "percent", "direction": "desc" }, "reproducibility": { "level": "fireworks", "label": "Fireworks", "statement": "Fireworks — the task set is held privately with Fireworks, which runs the evaluation." }, "source_url": "https://www.genspark.ai/blog/gen-1-slides", "source_title": "Open Genspark’s Gen-1 Slides evaluation", "methodology_url": "https://fireworks.ai/specialized-intelligence-index/#methodology-slide-generation%2Fgenspark-slides-benchmark", "methodology_summary": "Genspark evaluates eleven models in its production slide-generation harness on 200 real production tasks. The Specialized Intelligence Index publishes only the aggregate, the quality composite, four component scores, the layout penalty, the Tier A share, and the USD-per-deck values Genspark explicitly reports.", "board_note": null, "result_count": 11, "results": [ { "rank": 1, "ranked": true, "model": "Claude Fable 5.1", "provider": "Anthropic", "metrics": { "score": { "value": 83.59, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 74.29, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 87.8, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 81, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 68, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 76.9, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 7.26, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 50, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Fable 5.1", "provider": "Anthropic", "acc": 83.59, "qualityComposite": 74.29, "taskCompletion": 87.8, "contentQuality": 81, "visualDesign": 68, "processQuality": 76.9, "layoutPenalty": 7.26, "tierARatio": 50, "cost": null } }, { "rank": 2, "ranked": true, "model": "Gen-1 Slides", "provider": "Genspark", "metrics": { "score": { "value": 82.22, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": 0.44, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 75.21, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 77.8, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 73.4, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 75.8, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 72.2, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 21.52, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 71.4, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gen-1 Slides", "provider": "Genspark", "acc": 82.22, "qualityComposite": 75.21, "taskCompletion": 77.8, "contentQuality": 73.4, "visualDesign": 75.8, "processQuality": 72.2, "layoutPenalty": 21.52, "tierARatio": 71.4, "cost": 0.44, "specialist": true, "trainedWithFireworks": true } }, { "rank": 3, "ranked": true, "model": "Claude Opus 5", "provider": "Anthropic", "metrics": { "score": { "value": 81.02, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": 4.16, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 73.14, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 84.9, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 78.2, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 68.7, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 69.6, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 11.72, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 50.8, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Opus 5", "provider": "Anthropic", "acc": 81.02, "qualityComposite": 73.14, "taskCompletion": 84.9, "contentQuality": 78.2, "visualDesign": 68.7, "processQuality": 69.6, "layoutPenalty": 11.72, "tierARatio": 50.8, "cost": 4.16 } }, { "rank": 4, "ranked": true, "model": "Claude Fable 5", "provider": "Anthropic", "metrics": { "score": { "value": 79.92, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 70.53, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 87.2, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 81.1, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 61.4, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 77.1, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 12.13, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 40.7, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Claude Fable 5", "provider": "Anthropic", "acc": 79.92, "qualityComposite": 70.53, "taskCompletion": 87.2, "contentQuality": 81.1, "visualDesign": 61.4, "processQuality": 77.1, "layoutPenalty": 12.13, "tierARatio": 40.7, "cost": null } }, { "rank": 5, "ranked": true, "model": "GPT-6 Astra", "provider": "OpenAI", "metrics": { "score": { "value": 77.95, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 65.96, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 85, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 78.3, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 54.5, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 81.8, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 3.47, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 21.1, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-6 Astra", "provider": "OpenAI", "acc": 77.95, "qualityComposite": 65.96, "taskCompletion": 85, "contentQuality": 78.3, "visualDesign": 54.5, "processQuality": 81.8, "layoutPenalty": 3.47, "tierARatio": 21.1, "cost": null } }, { "rank": 6, "ranked": true, "model": "Kimi K3", "provider": "Moonshot AI", "metrics": { "score": { "value": 72.56, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": 2, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 66.21, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 84, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 78.4, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 55.9, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 74.9, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 21.67, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 34, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Kimi K3", "provider": "Moonshot AI", "acc": 72.56, "qualityComposite": 66.21, "taskCompletion": 84, "contentQuality": 78.4, "visualDesign": 55.9, "processQuality": 74.9, "layoutPenalty": 21.67, "tierARatio": 34, "cost": 2 } }, { "rank": 7, "ranked": true, "model": "GPT-5.6 Sol", "provider": "OpenAI", "metrics": { "score": { "value": 67.02, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": 2.01, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 60.58, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 82, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 74.2, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 48.2, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 74.9, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 21.31, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 22.3, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Sol", "provider": "OpenAI", "acc": 67.02, "qualityComposite": 60.58, "taskCompletion": 82, "contentQuality": 74.2, "visualDesign": 48.2, "processQuality": 74.9, "layoutPenalty": 21.31, "tierARatio": 22.3, "cost": 2.01 } }, { "rank": 8, "ranked": true, "model": "MiniMax M3", "provider": "MiniMax", "metrics": { "score": { "value": 56.07, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": 0.34, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 57.43, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 74, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 66.6, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 49.3, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 60.6, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 44.62, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 34.8, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "MiniMax M3", "provider": "MiniMax", "acc": 56.07, "qualityComposite": 57.43, "taskCompletion": 74, "contentQuality": 66.6, "visualDesign": 49.3, "processQuality": 60.6, "layoutPenalty": 44.62, "tierARatio": 34.8, "cost": 0.34 } }, { "rank": 9, "ranked": true, "model": "GPT-5.6 Luna", "provider": "OpenAI", "metrics": { "score": { "value": 54.06, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 52.21, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 74.9, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 65.6, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 39.7, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 65.8, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 33.26, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 18.4, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Luna", "provider": "OpenAI", "acc": 54.06, "qualityComposite": 52.21, "taskCompletion": 74.9, "contentQuality": 65.6, "visualDesign": 39.7, "processQuality": 65.8, "layoutPenalty": 33.26, "tierARatio": 18.4, "cost": null } }, { "rank": 10, "ranked": true, "model": "Gemini 3.8 Flash", "provider": "Google", "metrics": { "score": { "value": 53.39, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 52.98, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 79.4, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 74.9, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 35.8, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 66.9, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 38.92, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 6.9, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "Gemini 3.8 Flash", "provider": "Google", "acc": 53.39, "qualityComposite": 52.98, "taskCompletion": 79.4, "contentQuality": 74.9, "visualDesign": 35.8, "processQuality": 66.9, "layoutPenalty": 38.92, "tierARatio": 6.9, "cost": null } }, { "rank": 11, "ranked": true, "model": "GPT-5.6 Terra", "provider": "OpenAI", "metrics": { "score": { "value": 48.79, "unit": "percent", "label": "Score", "source_key": "acc" }, "cost_per_deck": { "value": null, "unit": "USD", "label": "Cost / deck", "source_key": "cost" }, "duration_per_task": { "value": null, "unit": "seconds", "label": "Duration / task", "source_key": "lat" }, "qualityComposite": { "value": 47.8, "unit": "percent", "label": "Score (quality composite)", "source_key": "qualityComposite" }, "taskCompletion": { "value": 71.1, "unit": "percent", "label": "Score (task completion)", "source_key": "taskCompletion" }, "contentQuality": { "value": 64.4, "unit": "percent", "label": "Score (content quality)", "source_key": "contentQuality" }, "visualDesign": { "value": 33.5, "unit": "percent", "label": "Score (visual design)", "source_key": "visualDesign" }, "processQuality": { "value": 63.1, "unit": "percent", "label": "Score (process quality)", "source_key": "processQuality" }, "layoutPenalty": { "value": 32.99, "unit": "percent", "label": "Penalty (layout defects)", "source_key": "layoutPenalty" }, "tierARatio": { "value": 6.8, "unit": "percent", "label": "Share (Tier A decks)", "source_key": "tierARatio" } }, "confidence_interval_95": null, "per_run_scores": null, "coverage": null, "note": null, "fireworks_model_url": null, "raw": { "model": "GPT-5.6 Terra", "provider": "OpenAI", "acc": 48.79, "qualityComposite": 47.8, "taskCompletion": 71.1, "contentQuality": 64.4, "visualDesign": 33.5, "processQuality": 63.1, "layoutPenalty": 32.99, "tierARatio": 6.8, "cost": null } } ] } ] }