{
 "chart": "make_eval/human_scores",
 "title": "AI agents now build borderline-useful tasks, but not yet ones we consider especially informative",
 "subtitle": "We show a METR researcher's rating of evaluation tasks built by models for assessment of AI capabilities.",
 "panels": [
  {
   "title": "",
   "x": {"label": "Model release date", "unit": "", "scale": "linear"},
   "y": {"label": "Human rating", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "runs",
     "rows": [
      {"model": "GPT-4o", "released": "2024-05-13", "run": "Run 1", "human_rating": 0.0},
      {"model": "GPT-4o", "released": "2024-05-13", "run": "Run 2", "human_rating": 0.01},
      {"model": "GPT-4o", "released": "2024-05-13", "run": "Run 3", "human_rating": 0.03},
      {"model": "o1", "released": "2024-12-05", "run": "Run 1", "human_rating": 0.0},
      {"model": "o1", "released": "2024-12-05", "run": "Run 2", "human_rating": 0.01},
      {"model": "o1", "released": "2024-12-05", "run": "Run 3", "human_rating": 0.0},
      {"model": "GPT-5", "released": "2025-08-07", "run": "Run 1", "human_rating": 0.05},
      {"model": "GPT-5", "released": "2025-08-07", "run": "Run 2", "human_rating": 0.01},
      {"model": "GPT-5", "released": "2025-08-07", "run": "Run 3", "human_rating": 0.02},
      {"model": "Claude Opus 4.5", "released": "2025-11-24", "run": "Run 1", "human_rating": 0.05},
      {"model": "Claude Opus 4.5", "released": "2025-11-24", "run": "Run 2", "human_rating": 0.15},
      {"model": "Claude Opus 4.5", "released": "2025-11-24", "run": "Run 3", "human_rating": 0.15},
      {"model": "Claude Opus 4.6", "released": "2026-02-05", "run": "Run 1", "human_rating": 0.07},
      {"model": "Claude Opus 4.6", "released": "2026-02-05", "run": "Run 2", "human_rating": 0.1},
      {"model": "Claude Opus 4.6", "released": "2026-02-05", "run": "Run 3", "human_rating": 0.35},
      {"model": "GPT-5.4", "released": "2026-03-05", "run": "Run 1", "human_rating": 0.03},
      {"model": "GPT-5.4", "released": "2026-03-05", "run": "Run 2", "human_rating": 0.27},
      {"model": "GPT-5.4", "released": "2026-03-05", "run": "Run 3", "human_rating": 0.15},
      {"model": "Claude Opus 4.7", "released": "2026-04-16", "run": "Run 1", "human_rating": 0.05},
      {"model": "Claude Opus 4.7", "released": "2026-04-16", "run": "Run 2", "human_rating": 0.08},
      {"model": "Claude Opus 4.7", "released": "2026-04-16", "run": "Run 3", "human_rating": 0.05},
      {"model": "GPT-5.5", "released": "2026-04-23", "run": "Run 1", "human_rating": 0.3},
      {"model": "GPT-5.5", "released": "2026-04-23", "run": "Run 2", "human_rating": 0.37},
      {"model": "GPT-5.5", "released": "2026-04-23", "run": "Run 3", "human_rating": 0.24},
      {"model": "Claude Opus 4.8", "released": "2026-05-28", "run": "Run 1", "human_rating": 0.35},
      {"model": "Claude Opus 4.8", "released": "2026-05-28", "run": "Run 2", "human_rating": 0.35},
      {"model": "Claude Opus 4.8", "released": "2026-05-28", "run": "Run 3", "human_rating": 0.33},
      {"model": "Claude Fable 5", "released": "2026-06-09", "run": "Run 1", "human_rating": 0.47},
      {"model": "Claude Fable 5", "released": "2026-06-09", "run": "Run 2", "human_rating": 0.58},
      {"model": "Claude Fable 5", "released": "2026-06-09", "run": "Run 3", "human_rating": 0.48},
      {"model": "GPT-5.6 Sol", "released": "2026-07-09", "run": "Run 1", "human_rating": 0.2},
      {"model": "GPT-5.6 Sol", "released": "2026-07-09", "run": "Run 2", "human_rating": 0.33},
      {"model": "GPT-5.6 Sol", "released": "2026-07-09", "run": "Run 3", "human_rating": 0.35},
      {"model": "Claude Opus 5", "released": "2026-07-24", "run": "Run 1", "human_rating": 0.3},
      {"model": "Claude Opus 5", "released": "2026-07-24", "run": "Run 2", "human_rating": 0.37},
      {"model": "Claude Opus 5", "released": "2026-07-24", "run": "Run 3", "human_rating": 0.48},
      {"model": "Claude Fable 5.1", "released": "2026-09-01", "run": "Run 1", "human_rating": 0.53},
      {"model": "Claude Fable 5.1", "released": "2026-09-01", "run": "Run 2", "human_rating": 0.26},
      {"model": "Claude Fable 5.1", "released": "2026-09-01", "run": "Run 3", "human_rating": 0.51},
      {"model": "GPT-6 Astra", "released": "2026-09-03", "run": "Run 1", "human_rating": 0.55},
      {"model": "GPT-6 Astra", "released": "2026-09-03", "run": "Run 2", "human_rating": 0.3},
      {"model": "GPT-6 Astra", "released": "2026-09-03", "run": "Run 3", "human_rating": 0.4},
      {"model": "Claude Opus 5.5", "released": "2026-09-22", "run": "Run 1", "human_rating": 0.54},
      {"model": "Claude Opus 5.5", "released": "2026-09-22", "run": "Run 2", "human_rating": 0.52},
      {"model": "Claude Opus 5.5", "released": "2026-09-22", "run": "Run 3", "human_rating": 0.55}
     ]
    },
    {
     "name": "models",
     "rows": [
      {"model": "GPT-4o", "released": "2024-05-13", "mean_human_rating": 0.0133333, "runs": 3},
      {"model": "o1", "released": "2024-12-05", "mean_human_rating": 0.00333333, "runs": 3},
      {"model": "GPT-5", "released": "2025-08-07", "mean_human_rating": 0.0266667, "runs": 3},
      {"model": "Claude Opus 4.5", "released": "2025-11-24", "mean_human_rating": 0.116667, "runs": 3},
      {"model": "Claude Opus 4.6", "released": "2026-02-05", "mean_human_rating": 0.173333, "runs": 3},
      {"model": "GPT-5.4", "released": "2026-03-05", "mean_human_rating": 0.15, "runs": 3},
      {"model": "Claude Opus 4.7", "released": "2026-04-16", "mean_human_rating": 0.06, "runs": 3},
      {"model": "GPT-5.5", "released": "2026-04-23", "mean_human_rating": 0.303333, "runs": 3},
      {"model": "Claude Opus 4.8", "released": "2026-05-28", "mean_human_rating": 0.343333, "runs": 3},
      {"model": "Claude Fable 5", "released": "2026-06-09", "mean_human_rating": 0.51, "runs": 3},
      {"model": "GPT-5.6 Sol", "released": "2026-07-09", "mean_human_rating": 0.293333, "runs": 3},
      {"model": "Claude Opus 5", "released": "2026-07-24", "mean_human_rating": 0.383333, "runs": 3},
      {"model": "Claude Fable 5.1", "released": "2026-09-01", "mean_human_rating": 0.433333, "runs": 3},
      {"model": "GPT-6 Astra", "released": "2026-09-03", "mean_human_rating": 0.416667, "runs": 3},
      {"model": "Claude Opus 5.5", "released": "2026-09-22", "mean_human_rating": 0.536667, "runs": 3}
     ]
    },
    {
     "name": "rating bands",
     "rows": [
      {"band": "Clearly above the bar", "meaning": "Informative for METR to run in risk assessments or predeployment evaluations, with under 10 hours of further staff work", "human_rating_from": 0.75, "human_rating_to": 1.0},
      {"band": "Borderline above the bar", "meaning": "A strong idea whose implementation needs more work, or a strong implementation of a lower-signal idea", "human_rating_from": 0.5, "human_rating_to": 0.75},
      {"band": "Borderline below the bar", "meaning": "Somewhat sensitive to capabilities we care about, but not worth further investment", "human_rating_from": 0.25, "human_rating_to": 0.5},
      {"band": "Clearly below the bar", "meaning": "Teaches us little about improving our risk assessments", "human_rating_from": 0, "human_rating_to": 0.25}
     ]
    }
   ]
  }
 ]
}
