{
 "chart": "swarm/tap_gap/tap_gap",
 "title": "Swarms overestimated their own performance in Train a Program",
 "subtitle": "",
 "panels": [
  {
   "title": "GPT-6 Astra · 2 agents",
   "x": {"label": "Hours from start", "unit": "", "scale": "linear"},
   "y": {"label": "Accuracy", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "hidden score over time",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 0.0, "hidden_test_score": 0.0},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 0.88502, "hidden_test_score": 0.0144755},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.15054, "hidden_test_score": 0.0925018},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.40719, "hidden_test_score": 0.0948535},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.64817, "hidden_test_score": 0.179046},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 3.50798, "hidden_test_score": 0.179253},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 4.37544, "hidden_test_score": 0.177611},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 5.61412, "hidden_test_score": 0.180971},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 6.1218, "hidden_test_score": 0.553104},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 7.01893, "hidden_test_score": 0.579617},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 7.63498, "hidden_test_score": 0.577455},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 7.71822, "hidden_test_score": 0.579276},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 8.11235, "hidden_test_score": 0.583052},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 8.99809, "hidden_test_score": 0.582128},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 10.0197, "hidden_test_score": 0.581876},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 11.4867, "hidden_test_score": 0.542069},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 12.2147, "hidden_test_score": 0.54595},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 14.097, "hidden_test_score": 0.547662},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 16.0699, "hidden_test_score": 0.548373},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 17.6277, "hidden_test_score": 0.537473},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 22.3529, "hidden_test_score": 0.534488},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 22.9235, "hidden_test_score": 0.548181},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 23.4194, "hidden_test_score": 0.548236}
     ]
    },
    {
     "name": "scores the team reported",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.15, "swarm_generated_test_set_accuracy": 0.729},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.41, "swarm_generated_test_set_accuracy": 0.755},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 2.84, "swarm_generated_test_set_accuracy": 0.798},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 4.38, "swarm_generated_test_set_accuracy": 0.82},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 6.12, "swarm_generated_test_set_accuracy": 0.696},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 7.02, "swarm_generated_test_set_accuracy": 0.71},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 11.49, "swarm_generated_test_set_accuracy": 0.715},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 12.21, "swarm_generated_test_set_accuracy": 0.729},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 17.63, "swarm_generated_test_set_accuracy": 0.729},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 19.1, "swarm_generated_test_set_accuracy": 0.731},
      {"model": "GPT-6 Astra", "team_size": 2, "hours_from_start": 22.35, "swarm_generated_test_set_accuracy": 0.742}
     ]
    },
    {
     "name": "reported score against final hidden score",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 2, "reported_hours_from_start": 22.35, "swarm_generated_test_set_accuracy": 0.742, "hidden_test_score": 0.548236, "reported_to_hidden_ratio": 1.35343}
     ]
    }
   ]
  },
  {
   "title": "GPT-6 Astra · 40 agents",
   "x": {"label": "Hours from start", "unit": "", "scale": "linear"},
   "y": {"label": "Accuracy", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "hidden score over time",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 0.0, "hidden_test_score": 0.0},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 0.26885, "hidden_test_score": 0.00883358},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 0.81358, "hidden_test_score": 0.0499162},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 1.32419, "hidden_test_score": 0.0499082},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 1.39976, "hidden_test_score": 0.0626836},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 1.58666, "hidden_test_score": 0.0626836},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 2.12915, "hidden_test_score": 0.0451512},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 2.26549, "hidden_test_score": 0.0451512},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 2.94041, "hidden_test_score": 0.0331882},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 4.1936, "hidden_test_score": 0.0164226},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 5.34437, "hidden_test_score": 0.0176325},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 6.0091, "hidden_test_score": 0.0176326},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 10.5908, "hidden_test_score": 0.0176326},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 18.7862, "hidden_test_score": 0.0185081},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 24.0, "hidden_test_score": 0.0185081}
     ]
    },
    {
     "name": "scores the team reported",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 0.81, "swarm_generated_test_set_accuracy": 0.805},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 1.32, "swarm_generated_test_set_accuracy": 0.816},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 1.4, "swarm_generated_test_set_accuracy": 0.789},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 2.27, "swarm_generated_test_set_accuracy": 0.789},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 2.94, "swarm_generated_test_set_accuracy": 0.92},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 4.19, "swarm_generated_test_set_accuracy": 0.92},
      {"model": "GPT-6 Astra", "team_size": 40, "hours_from_start": 5.34, "swarm_generated_test_set_accuracy": 0.92}
     ]
    },
    {
     "name": "reported score against final hidden score",
     "rows": [
      {"model": "GPT-6 Astra", "team_size": 40, "reported_hours_from_start": 5.34, "swarm_generated_test_set_accuracy": 0.92, "hidden_test_score": 0.0185081, "reported_to_hidden_ratio": 49.7079}
     ]
    }
   ]
  },
  {
   "title": "Claude Opus 5.5 · 2 agents",
   "x": {"label": "Hours from start", "unit": "", "scale": "linear"},
   "y": {"label": "Accuracy", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "hidden score over time",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 0.0, "hidden_test_score": 0.0},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 4.16287, "hidden_test_score": 0.1594},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 8.34023, "hidden_test_score": 0.309095},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 8.40013, "hidden_test_score": 0.351714},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 11.9873, "hidden_test_score": 0.406332},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 12.1682, "hidden_test_score": 0.416334},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 17.0188, "hidden_test_score": 0.423939},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 17.5006, "hidden_test_score": 0.44059},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 19.9356, "hidden_test_score": 0.477512},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 20.3212, "hidden_test_score": 0.48351},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 22.3625, "hidden_test_score": 0.485427},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 22.6731, "hidden_test_score": 0.498408},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 22.7926, "hidden_test_score": 0.498408}
     ]
    },
    {
     "name": "scores the team reported",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 1.42, "swarm_generated_test_set_accuracy": 0.043},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 4.16, "swarm_generated_test_set_accuracy": 0.351},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 8.34, "swarm_generated_test_set_accuracy": 0.394},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 8.4, "swarm_generated_test_set_accuracy": 0.423},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 11.99, "swarm_generated_test_set_accuracy": 0.468},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 12.17, "swarm_generated_test_set_accuracy": 0.513},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 17.02, "swarm_generated_test_set_accuracy": 0.555},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 17.5, "swarm_generated_test_set_accuracy": 0.545},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 19.94, "swarm_generated_test_set_accuracy": 0.554},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 20.32, "swarm_generated_test_set_accuracy": 0.575},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 22.36, "swarm_generated_test_set_accuracy": 0.58},
      {"model": "Claude Opus 5.5", "team_size": 2, "hours_from_start": 22.67, "swarm_generated_test_set_accuracy": 0.584}
     ]
    },
    {
     "name": "reported score against final hidden score",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 2, "reported_hours_from_start": 22.67, "swarm_generated_test_set_accuracy": 0.584, "hidden_test_score": 0.498408, "reported_to_hidden_ratio": 1.17173}
     ]
    }
   ]
  },
  {
   "title": "Claude Opus 5.5 · 40 agents",
   "x": {"label": "Hours from start", "unit": "", "scale": "linear"},
   "y": {"label": "Accuracy", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "hidden score over time",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 0.0, "hidden_test_score": 0.0},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 0.72778, "hidden_test_score": 0.0390788},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 3.09731, "hidden_test_score": 0.0651277},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 3.298, "hidden_test_score": 0.0651246},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 3.49253, "hidden_test_score": 0.0786908},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 4.82486, "hidden_test_score": 0.0928076},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 5.71936, "hidden_test_score": 0.0923893},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 5.91601, "hidden_test_score": 0.174732},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.68467, "hidden_test_score": 0.0966929},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.85703, "hidden_test_score": 0.0869259},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.92869, "hidden_test_score": 0.11516},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 7.48918, "hidden_test_score": 0.0836882},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 7.67737, "hidden_test_score": 0.165694},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 8.47919, "hidden_test_score": 0.14969},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 8.69281, "hidden_test_score": 0.156864},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 9.48473, "hidden_test_score": 0.175127},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 11.3873, "hidden_test_score": 0.154436},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 12.1802, "hidden_test_score": 0.137005},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 12.3747, "hidden_test_score": 0.109081},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 13.3767, "hidden_test_score": 0.108323},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 13.9632, "hidden_test_score": 0.104384},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 14.5312, "hidden_test_score": 0.106103},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 15.7167, "hidden_test_score": 0.106859},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 15.8137, "hidden_test_score": 0.106725},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 16.3249, "hidden_test_score": 0.106946},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 16.7124, "hidden_test_score": 0.108547},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 17.1266, "hidden_test_score": 0.110445},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 17.482, "hidden_test_score": 0.112924},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 18.4688, "hidden_test_score": 0.112828},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 19.0048, "hidden_test_score": 0.112494},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 19.2269, "hidden_test_score": 0.114344},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 20.5302, "hidden_test_score": 0.115667},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 20.9061, "hidden_test_score": 0.115952},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 21.8698, "hidden_test_score": 0.114921},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 22.7461, "hidden_test_score": 0.114985},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 22.8946, "hidden_test_score": 0.116405},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 24.0, "hidden_test_score": 0.116405}
     ]
    },
    {
     "name": "scores the team reported",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 0.73, "swarm_generated_test_set_accuracy": 0.33},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 3.1, "swarm_generated_test_set_accuracy": 0.417},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 3.49, "swarm_generated_test_set_accuracy": 0.687},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 4.82, "swarm_generated_test_set_accuracy": 0.665},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 5.72, "swarm_generated_test_set_accuracy": 0.621},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 5.92, "swarm_generated_test_set_accuracy": 0.656},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.68, "swarm_generated_test_set_accuracy": 0.673},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.86, "swarm_generated_test_set_accuracy": 0.683},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 6.93, "swarm_generated_test_set_accuracy": 0.692},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 7.49, "swarm_generated_test_set_accuracy": 0.704},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 7.68, "swarm_generated_test_set_accuracy": 0.771},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 8.48, "swarm_generated_test_set_accuracy": 0.761},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 9.48, "swarm_generated_test_set_accuracy": 0.796},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 11.39, "swarm_generated_test_set_accuracy": 0.821},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 12.37, "swarm_generated_test_set_accuracy": 0.824},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 13.38, "swarm_generated_test_set_accuracy": 0.831},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 14.53, "swarm_generated_test_set_accuracy": 0.85},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 15.72, "swarm_generated_test_set_accuracy": 0.855},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 15.81, "swarm_generated_test_set_accuracy": 0.862},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 17.48, "swarm_generated_test_set_accuracy": 0.885},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 18.47, "swarm_generated_test_set_accuracy": 0.888},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 19.0, "swarm_generated_test_set_accuracy": 0.893},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 19.23, "swarm_generated_test_set_accuracy": 0.9},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 20.91, "swarm_generated_test_set_accuracy": 0.908},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 21.87, "swarm_generated_test_set_accuracy": 0.911},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 22.75, "swarm_generated_test_set_accuracy": 0.912},
      {"model": "Claude Opus 5.5", "team_size": 40, "hours_from_start": 22.89, "swarm_generated_test_set_accuracy": 0.912}
     ]
    },
    {
     "name": "reported score against final hidden score",
     "rows": [
      {"model": "Claude Opus 5.5", "team_size": 40, "reported_hours_from_start": 22.89, "swarm_generated_test_set_accuracy": 0.912, "hidden_test_score": 0.116405, "reported_to_hidden_ratio": 7.83474}
     ]
    }
   ]
  }
 ]
}
