[ { "benchmark_id": "simplebench", "name": "SimpleBench", "version": "public site snapshot 2026-08-26", "construct": "Everyday reasoning and linguistic adversarial robustness", "item_stream": "frozen", "grader": "multiple-choice exact match", "environment": "one-shot", "update_rule": "versioned snapshot", "population": "benchmark-authored questions", "evaluation_mode": "fixed", "status": "active", "public_private": "More than 200 questions; public sample and dataset links available", "human_baseline": "83.7%, n=9", "license": "not_reported", "source_ids": [ "S01" ] }, { "benchmark_id": "swebench_verified", "name": "SWE-bench Verified", "version": "500-task verified split", "construct": "Resolve real software issues in repositories", "item_stream": "frozen public GitHub issues", "grader": "executable tests", "environment": "tool-using sandbox", "update_rule": "fixed split", "population": "12 open-source Python repositories", "evaluation_mode": "fixed", "status": "disputed", "public_private": "Public issue, repository, and test-derived tasks", "human_baseline": "not_reported", "license": "dataset terms vary by repository", "source_ids": [ "S04", "S05", "S06" ] }, { "benchmark_id": "arc_agi_2", "name": "ARC-AGI-2", "version": "2025 release, 120-task eval sets; 2026 competition metric", "construct": "Novel visual abstraction and compositional reasoning", "item_stream": "frozen versioned tasks", "grader": "whole-grid exact match", "environment": "one-shot visual task", "update_rule": "versioned public/semi-private/private sets", "population": "benchmark-authored tasks calibrated with public participants", "evaluation_mode": "fixed", "status": "active", "public_private": "1,000 public train; 120 public, 120 semi-private, 120 private eval", "human_baseline": "Every eval task solved pass@2 by at least two humans; 400+ participant calibration", "license": "Apache-2.0 public repository", "source_ids": [ "S07", "S08", "S09", "S55" ] }, { "benchmark_id": "chatbot_arena", "name": "Arena", "version": "multiple arenas; Text overall snapshot 2026-08-21", "construct": "A family of human-preference benchmarks: separate arenas compare text, code, web design, and other model outputs", "item_stream": "continuously user-generated", "grader": "live human pairwise preference", "environment": "arena-dependent chat, code, and rendered-output comparisons", "update_rule": "rolling leaderboards", "population": "self-selected Arena users and prompts", "evaluation_mode": "live", "status": "active", "public_private": "aggregated public leaderboards; data releases subject to policy", "human_baseline": "not_applicable", "license": "publisher terms", "source_ids": [ "S18", "S19", "S20", "S21", "S27", "S28" ] }, { "benchmark_id": "hle", "name": "Humanity's Last Exam", "version": "final 2,500-item set", "construct": "Closed-ended expert knowledge and reasoning", "item_stream": "frozen with held-out private set", "grader": "answer extraction plus automatic judge", "environment": "one-shot multimodal", "update_rule": "finalized version with private overfitting checks", "population": "global expert-authored questions", "evaluation_mode": "fixed", "status": "active", "public_private": "2,500 public questions plus private set", "human_baseline": "not_reported", "license": "dataset terms", "source_ids": [ "S10", "S11" ] }, { "benchmark_id": "gpqa_diamond", "name": "GPQA Diamond", "version": "198-question Diamond split", "construct": "Graduate-level science reasoning", "item_stream": "frozen public academic questions", "grader": "multiple-choice parsing", "environment": "one-shot, zero- or five-shot prompting", "update_rule": "fixed split", "population": "expert-validated science questions", "evaluation_mode": "fixed", "status": "saturated", "public_private": "public", "human_baseline": "expert/non-expert validation in original paper", "license": "dataset terms", "source_ids": [ "S13" ] }, { "benchmark_id": "mmlu_pro", "name": "MMLU Pro", "version": "Vals snapshot 2026-08-19", "construct": "Academic knowledge across 14 subjects", "item_stream": "frozen public dataset", "grader": "multiple-choice parsing", "environment": "five-shot chain-of-thought", "update_rule": "fixed split", "population": "more than 12,000 question-answer pairs", "evaluation_mode": "fixed", "status": "saturated", "public_private": "public", "human_baseline": "not_reported", "license": "dataset terms", "source_ids": [ "S16" ] }, { "benchmark_id": "livecodebench", "name": "LiveCodeBench", "version": "v6 as described by Vals", "construct": "Competitive programming and broader code capabilities", "item_stream": "periodically refreshed from contests", "grader": "hidden executable tests", "environment": "one-shot code generation", "update_rule": "continuous collection with dated releases", "population": "LeetCode, AtCoder, and Codeforces problems", "evaluation_mode": "refreshed", "status": "active", "public_private": "public prompts with hidden tests", "human_baseline": "not_reported", "license": "mixed source terms", "source_ids": [ "S17", "S30" ] }, { "benchmark_id": "livebench", "name": "LiveBench", "version": "paper v2", "construct": "Broad reasoning on recent information", "item_stream": "periodically refreshed from recent sources", "grader": "objective ground truth", "environment": "one-shot", "update_rule": "frequent releases", "population": "recent papers, news, datasets, and contests", "evaluation_mode": "refreshed", "status": "active", "public_private": "public releases", "human_baseline": "not_reported", "license": "repository terms", "source_ids": [ "S31" ] }, { "benchmark_id": "dynabench", "name": "Dynabench", "version": "paper release", "construct": "Model failures discovered through adversarial interaction", "item_stream": "human-and-model-in-the-loop", "grader": "task-dependent human labels", "environment": "interactive collection", "update_rule": "iterative rounds", "population": "crowd and expert contributors", "evaluation_mode": "live", "status": "historical method", "public_private": "task-dependent", "human_baseline": "not_reported", "license": "task-dependent", "source_ids": [ "S32" ] }, { "benchmark_id": "agent_arena", "name": "Agent Arena", "version": "methodology 2026-06-04", "construct": "Agent outcomes in real tool-using work", "item_stream": "continuously user-generated", "grader": "multiple observed outcome signals", "environment": "multi-turn tool-using agent", "update_rule": "rolling, time-decayed causal estimates", "population": "self-selected Agent Mode users and tasks", "evaluation_mode": "live", "status": "active", "public_private": "aggregated public results; session traces governed by policy", "human_baseline": "not_applicable", "license": "publisher terms", "source_ids": [ "S26" ] }, { "benchmark_id": "aa_intelligence_index", "name": "Artificial Analysis Intelligence Index", "version": "v4.1.1", "construct": "Composite of agentic, coding, scientific-reasoning, and general evaluations", "item_stream": "versioned composite of nine evaluations", "grader": "component-specific exact, executable, judge, and pairwise grading", "environment": "mixed one-shot and agentic tasks", "update_rule": "versioned index methodology", "population": "tasks from nine component evaluations", "evaluation_mode": "composite", "status": "active", "public_private": "mix of public and private component datasets", "human_baseline": "human expert anchor used in GDPval-AA v2; no universal composite human baseline reported", "license": "component-specific terms", "source_ids": [ "S37", "S38", "S39" ] }, { "benchmark_id": "arc_agi_3", "name": "ARC-AGI-3", "version": "2026; methodology checked 2026-09-16", "construct": "Learn unfamiliar interactive game rules and solve efficiently", "item_stream": "versioned games", "grader": "completion and squared action efficiency against first-time human baseline", "environment": "turn-based game environment", "update_rule": "versioned environments and methodology", "population": "human-authored games", "evaluation_mode": "fixed", "status": "active", "public_private": "Public examples and protected evaluation sets", "human_baseline": "Per-level upper median of completing first-time players", "license": "MIT public game; evaluation terms vary", "source_ids": [ "S61", "S62" ] }, { "benchmark_id": "epoch_eci", "name": "Epoch Capabilities Index (ECI)", "version": "methodology checked 2026-09-16", "construct": "Broad capability estimated across multiple benchmarks", "item_stream": "collected benchmark observations", "grader": "statistical fit across overlapping evaluations", "environment": "varies by underlying benchmark", "update_rule": "joint refit as evidence is incorporated", "population": "models with benchmark coverage", "evaluation_mode": "composite", "status": "active", "public_private": "Public methodology and index; underlying tests vary", "human_baseline": "not_applicable", "license": "publisher terms", "source_ids": [ "S66", "S67", "S68" ] } ]