{
 "chart": "irt/robustness_checks/grouped/weighting/comparison",
 "title": "Robustness to removing weighting",
 "subtitle": "",
 "panels": [
  {
   "title": "",
   "x": {"label": "Model release date", "unit": "", "scale": "linear"},
   "y": {"label": "MCI", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "models",
     "rows": [
      {"specification": "Baseline spec", "model": "GPT-4 0314", "released": "2023-03-14", "mci": -85.4341, "mci_low_95": -225.505, "mci_high_95": 2.78144},
      {"specification": "Baseline spec", "model": "GPT-4 1106", "released": "2023-11-06", "mci": -73.7458, "mci_low_95": -226.634, "mci_high_95": 35.0223},
      {"specification": "Baseline spec", "model": "Claude 3 Opus", "released": "2024-03-04", "mci": -84.4829, "mci_low_95": -223.083, "mci_high_95": 35.1119},
      {"specification": "Baseline spec", "model": "GPT-4 Turbo", "released": "2024-04-09", "mci": -105.64, "mci_low_95": -235.613, "mci_high_95": -1.24543},
      {"specification": "Baseline spec", "model": "GPT-4o", "released": "2024-05-13", "mci": 0.0, "mci_low_95": 0.0, "mci_high_95": 0.0},
      {"specification": "Baseline spec", "model": "Claude 3.5 Sonnet (Old)", "released": "2024-06-20", "mci": 70.5736, "mci_low_95": -29.6482, "mci_high_95": 170.932},
      {"specification": "Baseline spec", "model": "o1-preview", "released": "2024-09-12", "mci": 140.17, "mci_low_95": 3.896, "mci_high_95": 226.822},
      {"specification": "Baseline spec", "model": "Claude 3.5 Sonnet (New)", "released": "2024-10-22", "mci": 192.048, "mci_low_95": 76.6686, "mci_high_95": 285.467},
      {"specification": "Baseline spec", "model": "o1", "released": "2024-12-05", "mci": 237.395, "mci_low_95": 155.927, "mci_high_95": 326.311},
      {"specification": "Baseline spec", "model": "Claude 3.7 Sonnet", "released": "2025-02-24", "mci": 310.008, "mci_low_95": 239.645, "mci_high_95": 398.117},
      {"specification": "Baseline spec", "model": "o3", "released": "2025-04-16", "mci": 417.334, "mci_low_95": 316.637, "mci_high_95": 481.95},
      {"specification": "Baseline spec", "model": "Claude Opus 4", "released": "2025-05-22", "mci": 392.652, "mci_low_95": 319.445, "mci_high_95": 484.474},
      {"specification": "Baseline spec", "model": "Claude Opus 4.1", "released": "2025-08-05", "mci": 398.5, "mci_low_95": 331.555, "mci_high_95": 486.729},
      {"specification": "Baseline spec", "model": "GPT-5", "released": "2025-08-07", "mci": 478.578, "mci_low_95": 433.015, "mci_high_95": 543.257},
      {"specification": "Baseline spec", "model": "Gemini 3 Pro", "released": "2025-11-18", "mci": 490.476, "mci_low_95": 448.618, "mci_high_95": 550.21},
      {"specification": "Baseline spec", "model": "GPT-5.1-Codex-Max", "released": "2025-11-19", "mci": 477.713, "mci_low_95": 424.84, "mci_high_95": 536.979},
      {"specification": "Baseline spec", "model": "Claude Opus 4.5", "released": "2025-11-24", "mci": 533.254, "mci_low_95": 495.691, "mci_high_95": 581.772},
      {"specification": "Baseline spec", "model": "GPT-5.2", "released": "2025-12-11", "mci": 551.456, "mci_low_95": 515.852, "mci_high_95": 596.908},
      {"specification": "Baseline spec", "model": "Claude Opus 4.6", "released": "2026-02-05", "mci": 633.0, "mci_low_95": 633.0, "mci_high_95": 633.0},
      {"specification": "Baseline spec", "model": "GPT-5.3-Codex", "released": "2026-02-05", "mci": 566.461, "mci_low_95": 525.476, "mci_high_95": 623.826},
      {"specification": "Baseline spec", "model": "Gemini 3.1 Pro", "released": "2026-02-19", "mci": 568.914, "mci_low_95": 514.122, "mci_high_95": 633.933},
      {"specification": "Baseline spec", "model": "GPT-5.4", "released": "2026-03-05", "mci": 593.178, "mci_low_95": 559.687, "mci_high_95": 637.98},
      {"specification": "Baseline spec", "model": "Claude Mythos Preview (early)", "released": "2026-04-07", "mci": 671.221, "mci_low_95": 637.524, "mci_high_95": 761.37},
      {"specification": "Baseline spec", "model": "Claude Opus 4.7", "released": "2026-04-16", "mci": 631.225, "mci_low_95": 589.016, "mci_high_95": 688.69},
      {"specification": "Baseline spec", "model": "GPT-5.5", "released": "2026-04-23", "mci": 705.688, "mci_low_95": 657.833, "mci_high_95": 817.112},
      {"specification": "Baseline spec", "model": "Claude Opus 4.8", "released": "2026-05-28", "mci": 708.597, "mci_low_95": 661.3, "mci_high_95": 784.271},
      {"specification": "Baseline spec", "model": "Claude Fable 5", "released": "2026-06-09", "mci": 778.739, "mci_low_95": 712.935, "mci_high_95": 890.915},
      {"specification": "Baseline spec", "model": "GPT-5.6 Sol", "released": "2026-07-09", "mci": 738.275, "mci_low_95": 681.645, "mci_high_95": 855.844},
      {"specification": "Baseline spec", "model": "Claude Opus 5", "released": "2026-07-24", "mci": 796.229, "mci_low_95": 735.871, "mci_high_95": 910.133},
      {"specification": "Baseline spec", "model": "Claude Fable 5.1", "released": "2026-09-01", "mci": 839.121, "mci_low_95": 753.658, "mci_high_95": 1019.11},
      {"specification": "Baseline spec", "model": "GPT-6 Astra", "released": "2026-09-03", "mci": 858.977, "mci_low_95": 761.107, "mci_high_95": 1085.32},
      {"specification": "Baseline spec", "model": "Claude Opus 5.5", "released": "2026-09-22", "mci": 854.102, "mci_low_95": 765.156, "mci_high_95": 1069.3},
      {"specification": "Baseline spec", "model": "GPT-6 Luna", "released": "2026-09-22", "mci": 614.597, "mci_low_95": 577.603, "mci_high_95": 661.158},
      {"specification": "Baseline spec", "model": "Claude Haiku 5.5", "released": "2026-10-07", "mci": 627.801, "mci_low_95": 595.15, "mci_high_95": 688.857},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 0314", "released": "2023-03-14", "mci": -68.4673, "mci_low_95": -149.982, "mci_high_95": 7.23756},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 1106", "released": "2023-11-06", "mci": -57.8755, "mci_low_95": -130.638, "mci_high_95": 14.6763},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3 Opus", "released": "2024-03-04", "mci": -44.3677, "mci_low_95": -123.85, "mci_high_95": 15.9798},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 Turbo", "released": "2024-04-09", "mci": -66.1792, "mci_low_95": -164.722, "mci_high_95": 2.00411},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4o", "released": "2024-05-13", "mci": 0.0, "mci_low_95": 0.0, "mci_high_95": 0.0},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.5 Sonnet (Old)", "released": "2024-06-20", "mci": 71.9247, "mci_low_95": 13.0041, "mci_high_95": 136.663},
      {"specification": "No dispersion, family, or run weighting", "model": "o1-preview", "released": "2024-09-12", "mci": 142.157, "mci_low_95": 68.6971, "mci_high_95": 222.411},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.5 Sonnet (New)", "released": "2024-10-22", "mci": 144.879, "mci_low_95": 80.5756, "mci_high_95": 232.067},
      {"specification": "No dispersion, family, or run weighting", "model": "o1", "released": "2024-12-05", "mci": 247.479, "mci_low_95": 174.369, "mci_high_95": 349.183},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.7 Sonnet", "released": "2025-02-24", "mci": 290.26, "mci_low_95": 228.19, "mci_high_95": 366.22},
      {"specification": "No dispersion, family, or run weighting", "model": "o3", "released": "2025-04-16", "mci": 421.452, "mci_low_95": 353.956, "mci_high_95": 483.875},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4", "released": "2025-05-22", "mci": 374.173, "mci_low_95": 277.698, "mci_high_95": 455.425},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.1", "released": "2025-08-05", "mci": 389.665, "mci_low_95": 295.682, "mci_high_95": 464.403},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5", "released": "2025-08-07", "mci": 489.692, "mci_low_95": 446.974, "mci_high_95": 543.971},
      {"specification": "No dispersion, family, or run weighting", "model": "Gemini 3 Pro", "released": "2025-11-18", "mci": 507.752, "mci_low_95": 475.331, "mci_high_95": 563.193},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.1-Codex-Max", "released": "2025-11-19", "mci": 490.08, "mci_low_95": 445.999, "mci_high_95": 546.814},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.5", "released": "2025-11-24", "mci": 521.33, "mci_low_95": 478.403, "mci_high_95": 567.75},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.2", "released": "2025-12-11", "mci": 548.05, "mci_low_95": 508.332, "mci_high_95": 591.14},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.6", "released": "2026-02-05", "mci": 633.0, "mci_low_95": 633.0, "mci_high_95": 633.0},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.3-Codex", "released": "2026-02-05", "mci": 568.138, "mci_low_95": 534.633, "mci_high_95": 614.783},
      {"specification": "No dispersion, family, or run weighting", "model": "Gemini 3.1 Pro", "released": "2026-02-19", "mci": 592.995, "mci_low_95": 535.263, "mci_high_95": 643.578},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.4", "released": "2026-03-05", "mci": 582.375, "mci_low_95": 545.805, "mci_high_95": 637.512},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Mythos Preview (early)", "released": "2026-04-07", "mci": 661.277, "mci_low_95": 617.988, "mci_high_95": 726.789},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.7", "released": "2026-04-16", "mci": 610.232, "mci_low_95": 579.564, "mci_high_95": 669.263},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.5", "released": "2026-04-23", "mci": 694.17, "mci_low_95": 639.491, "mci_high_95": 784.976},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.8", "released": "2026-05-28", "mci": 669.813, "mci_low_95": 616.178, "mci_high_95": 748.152},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Fable 5", "released": "2026-06-09", "mci": 752.102, "mci_low_95": 694.503, "mci_high_95": 873.571},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.6 Sol", "released": "2026-07-09", "mci": 700.131, "mci_low_95": 618.419, "mci_high_95": 802.211},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 5", "released": "2026-07-24", "mci": 761.703, "mci_low_95": 693.718, "mci_high_95": 874.636},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Fable 5.1", "released": "2026-09-01", "mci": 809.916, "mci_low_95": 728.64, "mci_high_95": 968.102},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-6 Astra", "released": "2026-09-03", "mci": 825.085, "mci_low_95": 717.939, "mci_high_95": 988.779},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 5.5", "released": "2026-09-22", "mci": 810.49, "mci_low_95": 721.19, "mci_high_95": 1004.3},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-6 Luna", "released": "2026-09-22", "mci": 591.659, "mci_low_95": 536.187, "mci_high_95": 641.137},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Haiku 5.5", "released": "2026-10-07", "mci": 604.147, "mci_low_95": 549.896, "mci_high_95": 670.919}
     ]
    },
    {
     "name": "specifications",
     "rows": [
      {"specification": "Baseline spec", "bootstrap_draws_converged": 199, "bootstrap_draws_attempted": 200},
      {"specification": "No dispersion, family, or run weighting", "bootstrap_draws_converged": 200, "bootstrap_draws_attempted": 200}
     ]
    }
   ]
  },
  {
   "title": "",
   "x": {"label": "Baseline spec (MCI)", "unit": "", "scale": "linear"},
   "y": {"label": "Robustness spec (MCI)", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "models",
     "rows": [
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3 Opus", "baseline_mci": -84.4829, "baseline_mci_low_95": -223.083, "baseline_mci_high_95": 35.1119, "mci": -44.3677, "mci_low_95": -123.85, "mci_high_95": 15.9798},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.5 Sonnet (New)", "baseline_mci": 192.048, "baseline_mci_low_95": 76.6686, "baseline_mci_high_95": 285.467, "mci": 144.879, "mci_low_95": 80.5756, "mci_high_95": 232.067},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.5 Sonnet (Old)", "baseline_mci": 70.5736, "baseline_mci_low_95": -29.6482, "baseline_mci_high_95": 170.932, "mci": 71.9247, "mci_low_95": 13.0041, "mci_high_95": 136.663},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude 3.7 Sonnet", "baseline_mci": 310.008, "baseline_mci_low_95": 239.645, "baseline_mci_high_95": 398.117, "mci": 290.26, "mci_low_95": 228.19, "mci_high_95": 366.22},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Fable 5", "baseline_mci": 778.739, "baseline_mci_low_95": 712.935, "baseline_mci_high_95": 890.915, "mci": 752.102, "mci_low_95": 694.503, "mci_high_95": 873.571},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Fable 5.1", "baseline_mci": 839.121, "baseline_mci_low_95": 753.658, "baseline_mci_high_95": 1019.11, "mci": 809.916, "mci_low_95": 728.64, "mci_high_95": 968.102},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Haiku 5.5", "baseline_mci": 627.801, "baseline_mci_low_95": 595.15, "baseline_mci_high_95": 688.857, "mci": 604.147, "mci_low_95": 549.896, "mci_high_95": 670.919},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Mythos Preview (early)", "baseline_mci": 671.221, "baseline_mci_low_95": 637.524, "baseline_mci_high_95": 761.37, "mci": 661.277, "mci_low_95": 617.988, "mci_high_95": 726.789},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4", "baseline_mci": 392.652, "baseline_mci_low_95": 319.445, "baseline_mci_high_95": 484.474, "mci": 374.173, "mci_low_95": 277.698, "mci_high_95": 455.425},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.1", "baseline_mci": 398.5, "baseline_mci_low_95": 331.555, "baseline_mci_high_95": 486.729, "mci": 389.665, "mci_low_95": 295.682, "mci_high_95": 464.403},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.5", "baseline_mci": 533.254, "baseline_mci_low_95": 495.691, "baseline_mci_high_95": 581.772, "mci": 521.33, "mci_low_95": 478.403, "mci_high_95": 567.75},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.6", "baseline_mci": 633.0, "baseline_mci_low_95": 633.0, "baseline_mci_high_95": 633.0, "mci": 633.0, "mci_low_95": 633.0, "mci_high_95": 633.0},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.7", "baseline_mci": 631.225, "baseline_mci_low_95": 589.016, "baseline_mci_high_95": 688.69, "mci": 610.232, "mci_low_95": 579.564, "mci_high_95": 669.263},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 4.8", "baseline_mci": 708.597, "baseline_mci_low_95": 661.3, "baseline_mci_high_95": 784.271, "mci": 669.813, "mci_low_95": 616.178, "mci_high_95": 748.152},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 5", "baseline_mci": 796.229, "baseline_mci_low_95": 735.871, "baseline_mci_high_95": 910.133, "mci": 761.703, "mci_low_95": 693.718, "mci_high_95": 874.636},
      {"specification": "No dispersion, family, or run weighting", "model": "Claude Opus 5.5", "baseline_mci": 854.102, "baseline_mci_low_95": 765.156, "baseline_mci_high_95": 1069.3, "mci": 810.49, "mci_low_95": 721.19, "mci_high_95": 1004.3},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 0314", "baseline_mci": -85.4341, "baseline_mci_low_95": -225.505, "baseline_mci_high_95": 2.78144, "mci": -68.4673, "mci_low_95": -149.982, "mci_high_95": 7.23756},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 1106", "baseline_mci": -73.7458, "baseline_mci_low_95": -226.634, "baseline_mci_high_95": 35.0223, "mci": -57.8755, "mci_low_95": -130.638, "mci_high_95": 14.6763},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4 Turbo", "baseline_mci": -105.64, "baseline_mci_low_95": -235.613, "baseline_mci_high_95": -1.24543, "mci": -66.1792, "mci_low_95": -164.722, "mci_high_95": 2.00411},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-4o", "baseline_mci": 0.0, "baseline_mci_low_95": 0.0, "baseline_mci_high_95": 0.0, "mci": 0.0, "mci_low_95": 0.0, "mci_high_95": 0.0},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5", "baseline_mci": 478.578, "baseline_mci_low_95": 433.015, "baseline_mci_high_95": 543.257, "mci": 489.692, "mci_low_95": 446.974, "mci_high_95": 543.971},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.1-Codex-Max", "baseline_mci": 477.713, "baseline_mci_low_95": 424.84, "baseline_mci_high_95": 536.979, "mci": 490.08, "mci_low_95": 445.999, "mci_high_95": 546.814},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.2", "baseline_mci": 551.456, "baseline_mci_low_95": 515.852, "baseline_mci_high_95": 596.908, "mci": 548.05, "mci_low_95": 508.332, "mci_high_95": 591.14},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.3-Codex", "baseline_mci": 566.461, "baseline_mci_low_95": 525.476, "baseline_mci_high_95": 623.826, "mci": 568.138, "mci_low_95": 534.633, "mci_high_95": 614.783},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.4", "baseline_mci": 593.178, "baseline_mci_low_95": 559.687, "baseline_mci_high_95": 637.98, "mci": 582.375, "mci_low_95": 545.805, "mci_high_95": 637.512},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.5", "baseline_mci": 705.688, "baseline_mci_low_95": 657.833, "baseline_mci_high_95": 817.112, "mci": 694.17, "mci_low_95": 639.491, "mci_high_95": 784.976},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-5.6 Sol", "baseline_mci": 738.275, "baseline_mci_low_95": 681.645, "baseline_mci_high_95": 855.844, "mci": 700.131, "mci_low_95": 618.419, "mci_high_95": 802.211},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-6 Astra", "baseline_mci": 858.977, "baseline_mci_low_95": 761.107, "baseline_mci_high_95": 1085.32, "mci": 825.085, "mci_low_95": 717.939, "mci_high_95": 988.779},
      {"specification": "No dispersion, family, or run weighting", "model": "GPT-6 Luna", "baseline_mci": 614.597, "baseline_mci_low_95": 577.603, "baseline_mci_high_95": 661.158, "mci": 591.659, "mci_low_95": 536.187, "mci_high_95": 641.137},
      {"specification": "No dispersion, family, or run weighting", "model": "Gemini 3 Pro", "baseline_mci": 490.476, "baseline_mci_low_95": 448.618, "baseline_mci_high_95": 550.21, "mci": 507.752, "mci_low_95": 475.331, "mci_high_95": 563.193},
      {"specification": "No dispersion, family, or run weighting", "model": "Gemini 3.1 Pro", "baseline_mci": 568.914, "baseline_mci_low_95": 514.122, "baseline_mci_high_95": 633.933, "mci": 592.995, "mci_low_95": 535.263, "mci_high_95": 643.578},
      {"specification": "No dispersion, family, or run weighting", "model": "o1", "baseline_mci": 237.395, "baseline_mci_low_95": 155.927, "baseline_mci_high_95": 326.311, "mci": 247.479, "mci_low_95": 174.369, "mci_high_95": 349.183},
      {"specification": "No dispersion, family, or run weighting", "model": "o1-preview", "baseline_mci": 140.17, "baseline_mci_low_95": 3.896, "baseline_mci_high_95": 226.822, "mci": 142.157, "mci_low_95": 68.6971, "mci_high_95": 222.411},
      {"specification": "No dispersion, family, or run weighting", "model": "o3", "baseline_mci": 417.334, "baseline_mci_low_95": 316.637, "baseline_mci_high_95": 481.95, "mci": 421.452, "mci_low_95": 353.956, "mci_high_95": 483.875}
     ]
    },
    {
     "name": "reference lines",
     "rows": [
      {"reference_line": "equal MCI under both specifications", "slope": 1, "intercept": 0}
     ]
    }
   ]
  }
 ]
}
