{
 "chart": "cost_subset/irt/expected_scores",
 "title": "On our dataset, the IRT model predicts lower scores on tasks that we estimate to be costlier",
 "subtitle": "We fit one IRT model on all tasks and all models. Points are a model's expected score, averaged over the tasks in one subset. Bars are 80% CIs over bootstrap draws.",
 "panels": [
  {
   "title": "",
   "x": {"label": "Model release date", "unit": "", "scale": "linear"},
   "y": {"label": "IRT expected score (log-odds scale)", "unit": "", "scale": "logit"},
   "tables": [
    {
     "name": "models",
     "rows": [
      {"subset": "Lower relative cost", "model": "GPT-4o", "released": "2024-05-13", "expected_score": 0.558144, "expected_score_low_80": 0.4128, "expected_score_high_80": 0.657585},
      {"subset": "Lower relative cost", "model": "o1", "released": "2024-12-05", "expected_score": 0.705642, "expected_score_low_80": 0.615668, "expected_score_high_80": 0.778087},
      {"subset": "Lower relative cost", "model": "GPT-5", "released": "2025-08-07", "expected_score": 0.83676, "expected_score_low_80": 0.795792, "expected_score_high_80": 0.881477},
      {"subset": "Lower relative cost", "model": "Claude Opus 4.5", "released": "2025-11-24", "expected_score": 0.860851, "expected_score_low_80": 0.821758, "expected_score_high_80": 0.897083},
      {"subset": "Lower relative cost", "model": "Claude Opus 4.6", "released": "2026-02-05", "expected_score": 0.897622, "expected_score_low_80": 0.860373, "expected_score_high_80": 0.925564},
      {"subset": "Lower relative cost", "model": "GPT-5.4", "released": "2026-03-05", "expected_score": 0.884639, "expected_score_low_80": 0.846657, "expected_score_high_80": 0.917032},
      {"subset": "Lower relative cost", "model": "Claude Opus 4.7", "released": "2026-04-16", "expected_score": 0.897093, "expected_score_low_80": 0.861651, "expected_score_high_80": 0.925134},
      {"subset": "Lower relative cost", "model": "GPT-5.5", "released": "2026-04-23", "expected_score": 0.91552, "expected_score_low_80": 0.879912, "expected_score_high_80": 0.940874},
      {"subset": "Lower relative cost", "model": "Claude Opus 4.8", "released": "2026-05-28", "expected_score": 0.916104, "expected_score_low_80": 0.882214, "expected_score_high_80": 0.942065},
      {"subset": "Lower relative cost", "model": "Claude Fable 5", "released": "2026-06-09", "expected_score": 0.928419, "expected_score_low_80": 0.896533, "expected_score_high_80": 0.951326},
      {"subset": "Lower relative cost", "model": "GPT-5.6 Sol", "released": "2026-07-09", "expected_score": 0.921664, "expected_score_low_80": 0.889322, "expected_score_high_80": 0.947124},
      {"subset": "Lower relative cost", "model": "Claude Opus 5", "released": "2026-07-24", "expected_score": 0.931179, "expected_score_low_80": 0.90137, "expected_score_high_80": 0.953804},
      {"subset": "Lower relative cost", "model": "Claude Fable 5.1", "released": "2026-09-01", "expected_score": 0.937874, "expected_score_low_80": 0.912347, "expected_score_high_80": 0.959501},
      {"subset": "Lower relative cost", "model": "GPT-6 Astra", "released": "2026-09-03", "expected_score": 0.941041, "expected_score_low_80": 0.9141, "expected_score_high_80": 0.964035},
      {"subset": "Lower relative cost", "model": "Claude Opus 5.5", "released": "2026-09-22", "expected_score": 0.940256, "expected_score_low_80": 0.913768, "expected_score_high_80": 0.963339},
      {"subset": "Higher relative cost", "model": "GPT-4o", "released": "2024-05-13", "expected_score": 0.487711, "expected_score_low_80": 0.381263, "expected_score_high_80": 0.5852},
      {"subset": "Higher relative cost", "model": "o1", "released": "2024-12-05", "expected_score": 0.616907, "expected_score_low_80": 0.523528, "expected_score_high_80": 0.703622},
      {"subset": "Higher relative cost", "model": "GPT-5", "released": "2025-08-07", "expected_score": 0.742051, "expected_score_low_80": 0.683363, "expected_score_high_80": 0.812778},
      {"subset": "Higher relative cost", "model": "Claude Opus 4.5", "released": "2025-11-24", "expected_score": 0.769609, "expected_score_low_80": 0.70933, "expected_score_high_80": 0.826767},
      {"subset": "Higher relative cost", "model": "Claude Opus 4.6", "released": "2026-02-05", "expected_score": 0.819721, "expected_score_low_80": 0.761598, "expected_score_high_80": 0.872711},
      {"subset": "Higher relative cost", "model": "GPT-5.4", "released": "2026-03-05", "expected_score": 0.799321, "expected_score_low_80": 0.750029, "expected_score_high_80": 0.857474},
      {"subset": "Higher relative cost", "model": "Claude Opus 4.7", "released": "2026-04-16", "expected_score": 0.818792, "expected_score_low_80": 0.769203, "expected_score_high_80": 0.869765},
      {"subset": "Higher relative cost", "model": "GPT-5.5", "released": "2026-04-23", "expected_score": 0.856094, "expected_score_low_80": 0.8118, "expected_score_high_80": 0.904497},
      {"subset": "Higher relative cost", "model": "Claude Opus 4.8", "released": "2026-05-28", "expected_score": 0.857345, "expected_score_low_80": 0.809299, "expected_score_high_80": 0.902067},
      {"subset": "Higher relative cost", "model": "Claude Fable 5", "released": "2026-06-09", "expected_score": 0.881713, "expected_score_low_80": 0.837666, "expected_score_high_80": 0.921335},
      {"subset": "Higher relative cost", "model": "GPT-5.6 Sol", "released": "2026-07-09", "expected_score": 0.868894, "expected_score_low_80": 0.822394, "expected_score_high_80": 0.917354},
      {"subset": "Higher relative cost", "model": "Claude Opus 5", "released": "2026-07-24", "expected_score": 0.886494, "expected_score_low_80": 0.846719, "expected_score_high_80": 0.929189},
      {"subset": "Higher relative cost", "model": "Claude Fable 5.1", "released": "2026-09-01", "expected_score": 0.896904, "expected_score_low_80": 0.858543, "expected_score_high_80": 0.938292},
      {"subset": "Higher relative cost", "model": "GPT-6 Astra", "released": "2026-09-03", "expected_score": 0.913555, "expected_score_low_80": 0.879465, "expected_score_high_80": 0.95405},
      {"subset": "Higher relative cost", "model": "Claude Opus 5.5", "released": "2026-09-22", "expected_score": 0.900213, "expected_score_low_80": 0.864197, "expected_score_high_80": 0.940052}
     ]
    },
    {
     "name": "subsets",
     "rows": [
      {"subset": "Lower relative cost", "tasks": 101},
      {"subset": "Higher relative cost", "tasks": 81}
     ]
    }
   ]
  }
 ]
}
