{
  "plan_id": "codex-heavy-analysis-2026-11-07-v2-0.1.5",
  "run": "codex/2026-11-07-heavy-jail",
  "attempts": 116,
  "pairs": 58,
  "plugin_versions": [
    "0.1.5"
  ],
  "suites": {
    "c6": {
      "name": "C6 Codex-native heavy long sessions (the Claude heavy run's S6 tasks, goal-stating turns, 14 or 15 scripted turns)",
      "pairs_run": 44,
      "valid_pairs": 43,
      "invalid": [
        {
          "pair_id": "J005-c6-sympy-sets-intervals-r1-ab",
          "reason": "on usage incomplete; on stopped"
        }
      ],
      "primary_non_cached": {
        "pairs": 43,
        "control_sum": 15855591,
        "treatment_sum": 14475644,
        "pooled_change_pct": -8.7,
        "ci95_pct": [
          -16.1,
          -1.3
        ],
        "per_pair_median_pct": -6.0,
        "share_of_pairs_lower": 0.605,
        "signflip_p": 0.072946,
        "signflip_method": "monte-carlo"
      },
      "secondary": {
        "total": {
          "pairs": 43,
          "control_sum": 729010919,
          "treatment_sum": 766918012,
          "pooled_change_pct": 5.2,
          "ci95_pct": [
            -6.2,
            17.3
          ],
          "per_pair_median_pct": 0.2,
          "share_of_pairs_lower": 0.465,
          "signflip_p": 0.412879,
          "signflip_method": "monte-carlo"
        },
        "cached": {
          "pairs": 43,
          "control_sum": 713155328,
          "treatment_sum": 752442368,
          "pooled_change_pct": 5.5,
          "ci95_pct": [
            -6.1,
            17.9
          ],
          "per_pair_median_pct": 0.5,
          "share_of_pairs_lower": 0.442,
          "signflip_p": 0.39183,
          "signflip_method": "monte-carlo"
        },
        "cost": {
          "pairs": 43,
          "control_sum": 114.0614,
          "treatment_sum": 117.0609,
          "pooled_change_pct": 2.6,
          "ci95_pct": [
            -6.4,
            12.1
          ],
          "per_pair_median_pct": 1.4,
          "share_of_pairs_lower": 0.442,
          "signflip_p": 0.59327,
          "signflip_method": "monte-carlo"
        },
        "wall": {
          "pairs": 43,
          "control_sum": 40933.8,
          "treatment_sum": 52651.0,
          "pooled_change_pct": 28.6,
          "ci95_pct": [
            18.9,
            39.0
          ],
          "per_pair_median_pct": 28.3,
          "share_of_pairs_lower": 0.186,
          "signflip_p": 5e-05,
          "signflip_method": "monte-carlo"
        },
        "turns": {
          "pairs": 43,
          "control_sum": 615,
          "treatment_sum": 615,
          "pooled_change_pct": 0.0,
          "ci95_pct": [
            0.0,
            0.0
          ],
          "per_pair_median_pct": 0.0,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "monte-carlo"
        }
      },
      "categories": {
        "input_tokens": {
          "pairs": 43,
          "control_sum": 14476290,
          "treatment_sum": 12867496,
          "pooled_change_pct": -11.1,
          "ci95_pct": [
            -18.8,
            -3.4
          ],
          "per_pair_median_pct": -10.5,
          "share_of_pairs_lower": 0.628,
          "signflip_p": 0.028399,
          "signflip_method": "monte-carlo"
        },
        "cache_creation_input_tokens": {
          "pairs": 43,
          "control_sum": 0,
          "treatment_sum": 0,
          "pooled_change_pct": null,
          "ci95_pct": null,
          "per_pair_median_pct": null,
          "share_of_pairs_lower": null,
          "signflip_p": 1.0,
          "signflip_method": "monte-carlo"
        },
        "cache_read_input_tokens": {
          "pairs": 43,
          "control_sum": 713155328,
          "treatment_sum": 752442368,
          "pooled_change_pct": 5.5,
          "ci95_pct": [
            -6.1,
            17.9
          ],
          "per_pair_median_pct": 0.5,
          "share_of_pairs_lower": 0.442,
          "signflip_p": 0.39183,
          "signflip_method": "monte-carlo"
        },
        "output_tokens": {
          "pairs": 43,
          "control_sum": 1379301,
          "treatment_sum": 1608148,
          "pooled_change_pct": 16.6,
          "ci95_pct": [
            8.8,
            24.5
          ],
          "per_pair_median_pct": 19.4,
          "share_of_pairs_lower": 0.279,
          "signflip_p": 0.0001,
          "signflip_method": "monte-carlo"
        }
      },
      "quality": {
        "graded_pairs": 43,
        "control_accepted": 38,
        "treatment_accepted": 37,
        "control_only": 1,
        "treatment_only": 0,
        "mcnemar_exact_p": 1.0,
        "pass_rate_diff_points": -2.3,
        "ci95_points": [
          -6.8,
          2.2
        ],
        "non_inferior_at_minus_5": false
      },
      "pass_rate": {
        "off": {
          "accepted": 38,
          "graded": 43,
          "rate_pct": 88.4
        },
        "on": {
          "accepted": 37,
          "graded": 43,
          "rate_pct": 86.0
        }
      },
      "trims_on": 297,
      "trims_off": 0,
      "levers_on": {
        "test": {
          "events": 297,
          "plugin_estimated_tokens_removed": 4362212
        }
      },
      "inert_on_pairs": 0,
      "lever_activity_on": {
        "sessions": 43,
        "sessions_compaction_fired": 43,
        "sessions_first_output_context": 43,
        "sessions_any_replacement": 43,
        "replacements_by_kind": {
          "first-output-context": 43,
          "test": 297
        },
        "hook_reasons": {
          "source filter off": 609,
          "unsupported command": 1449,
          "off, small, or unsupported": 403,
          "no text response or tool input": 493,
          "search filter off": 572,
          "script-log filter off": 142,
          "token savings unverified or below overhead reserve": 64,
          "required evidence or recovery overhead exceeds savings": 17,
          "read filter off": 6
        },
        "bash_results": 4052,
        "plugin_stats_events": 297,
        "plugin_estimated_tokens_removed": 4362212
      },
      "lever_activity_off": {
        "sessions": 43,
        "sessions_compaction_fired": 0,
        "sessions_first_output_context": 0,
        "sessions_any_replacement": 0,
        "replacements_by_kind": {},
        "hook_reasons": {
          "unsupported command": 1026,
          "source filter off": 599,
          "off, small, or unsupported": 647,
          "no text response or tool input": 388,
          "script-log filter off": 127,
          "search filter off": 527,
          "read filter off": 1
        },
        "bash_results": 3315,
        "plugin_stats_events": 0,
        "plugin_estimated_tokens_removed": 0
      },
      "aa": {
        "pairs": 14,
        "control_sum": 4840713,
        "treatment_sum": 4787986,
        "pooled_change_pct": -1.1,
        "ci95_pct": [
          -8.4,
          5.7
        ],
        "per_pair_median_pct": -0.1,
        "share_of_pairs_lower": 0.5,
        "signflip_p": 0.88269,
        "signflip_method": "exact"
      },
      "aa_pairs": 14,
      "aa_invalid": [],
      "inert_by_task": {
        "django-cache-headers": {
          "pairs": 5,
          "on_arm_no_reduction": 0
        },
        "django-forms-fields": {
          "pairs": 5,
          "on_arm_no_reduction": 0
        },
        "django-http-request": {
          "pairs": 4,
          "on_arm_no_reduction": 0
        },
        "django-template-filters": {
          "pairs": 4,
          "on_arm_no_reduction": 0
        },
        "django-urls-reverse": {
          "pairs": 4,
          "on_arm_no_reduction": 0
        },
        "sympy-iterables": {
          "pairs": 4,
          "on_arm_no_reduction": 0
        },
        "sympy-latex-printing": {
          "pairs": 5,
          "on_arm_no_reduction": 0
        },
        "sympy-ntheory": {
          "pairs": 4,
          "on_arm_no_reduction": 0
        },
        "sympy-permutations": {
          "pairs": 5,
          "on_arm_no_reduction": 0
        },
        "sympy-sets-intervals": {
          "pairs": 3,
          "on_arm_no_reduction": 0
        }
      },
      "aa_total": {
        "pairs": 14,
        "control_sum": 225813641,
        "treatment_sum": 233189650,
        "pooled_change_pct": 3.3,
        "ci95_pct": [
          -8.6,
          18.9
        ],
        "per_pair_median_pct": -0.5,
        "share_of_pairs_lower": 0.571,
        "signflip_p": 0.575684,
        "signflip_method": "exact"
      },
      "relay_undercount": {
        "off": {
          "attempts": 43,
          "requests": 4234,
          "abandoned": 0,
          "attempts_with_abandoned": 0,
          "hidden_upper_non_cached": 0,
          "tap_agrees_with_client": 43,
          "ws_requests": 4234,
          "http_requests": 0
        },
        "on": {
          "attempts": 43,
          "requests": 4862,
          "abandoned": 2,
          "attempts_with_abandoned": 2,
          "hidden_upper_non_cached": 558738,
          "tap_agrees_with_client": 43,
          "ws_requests": 4862,
          "http_requests": 0
        },
        "non_cached_with_hidden_upper": {
          "pairs": 43,
          "control_sum": 15855591,
          "treatment_sum": 15034382,
          "pooled_change_pct": -5.2,
          "ci95_pct": [
            -13.8,
            3.6
          ],
          "per_pair_median_pct": -6.0,
          "share_of_pairs_lower": 0.605,
          "signflip_p": 0.354382,
          "signflip_method": "monte-carlo"
        }
      },
      "per_task": {
        "django-cache-headers": {
          "pairs": 5,
          "control_sum": 2720162,
          "treatment_sum": 1896986,
          "pooled_change_pct": -30.3,
          "ci95_pct": [
            -48.5,
            -7.2
          ],
          "per_pair_median_pct": -10.4,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 0.0625,
          "signflip_method": "exact"
        },
        "django-forms-fields": {
          "pairs": 5,
          "control_sum": 1454626,
          "treatment_sum": 1217053,
          "pooled_change_pct": -16.3,
          "ci95_pct": [
            -34.7,
            1.6
          ],
          "per_pair_median_pct": -10.7,
          "share_of_pairs_lower": 0.8,
          "signflip_p": 0.25,
          "signflip_method": "exact"
        },
        "django-http-request": {
          "pairs": 4,
          "control_sum": 1090389,
          "treatment_sum": 878165,
          "pooled_change_pct": -19.5,
          "ci95_pct": [
            -30.1,
            -13.6
          ],
          "per_pair_median_pct": -14.7,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 0.125,
          "signflip_method": "exact"
        },
        "django-template-filters": {
          "pairs": 4,
          "control_sum": 1880651,
          "treatment_sum": 1563246,
          "pooled_change_pct": -16.9,
          "ci95_pct": [
            -42.0,
            10.0
          ],
          "per_pair_median_pct": -6.2,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.625,
          "signflip_method": "exact"
        },
        "django-urls-reverse": {
          "pairs": 4,
          "control_sum": 1035830,
          "treatment_sum": 936409,
          "pooled_change_pct": -9.6,
          "ci95_pct": [
            -25.7,
            7.8
          ],
          "per_pair_median_pct": -4.6,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.625,
          "signflip_method": "exact"
        },
        "sympy-iterables": {
          "pairs": 4,
          "control_sum": 1307212,
          "treatment_sum": 1168501,
          "pooled_change_pct": -10.6,
          "ci95_pct": [
            -24.1,
            2.9
          ],
          "per_pair_median_pct": -6.0,
          "share_of_pairs_lower": 0.75,
          "signflip_p": 0.375,
          "signflip_method": "exact"
        },
        "sympy-latex-printing": {
          "pairs": 5,
          "control_sum": 2370981,
          "treatment_sum": 2596069,
          "pooled_change_pct": 9.5,
          "ci95_pct": [
            -11.0,
            23.8
          ],
          "per_pair_median_pct": 20.4,
          "share_of_pairs_lower": 0.2,
          "signflip_p": 0.4375,
          "signflip_method": "exact"
        },
        "sympy-ntheory": {
          "pairs": 4,
          "control_sum": 1084666,
          "treatment_sum": 1246147,
          "pooled_change_pct": 14.9,
          "ci95_pct": [
            0.4,
            24.4
          ],
          "per_pair_median_pct": 23.0,
          "share_of_pairs_lower": 0.25,
          "signflip_p": 0.25,
          "signflip_method": "exact"
        },
        "sympy-permutations": {
          "pairs": 5,
          "control_sum": 1871124,
          "treatment_sum": 1886640,
          "pooled_change_pct": 0.8,
          "ci95_pct": [
            -14.7,
            18.7
          ],
          "per_pair_median_pct": -5.4,
          "share_of_pairs_lower": 0.6,
          "signflip_p": 0.9375,
          "signflip_method": "exact"
        },
        "sympy-sets-intervals": {
          "pairs": 3,
          "control_sum": 1039950,
          "treatment_sum": 1086428,
          "pooled_change_pct": 4.5,
          "ci95_pct": [
            -9.0,
            12.4
          ],
          "per_pair_median_pct": 10.7,
          "share_of_pairs_lower": 0.333,
          "signflip_p": 0.5,
          "signflip_method": "exact"
        }
      },
      "rounds": {
        "1": {
          "pairs": 9,
          "control_sum": 3237408,
          "treatment_sum": 3229890,
          "pooled_change_pct": -0.2,
          "ci95_pct": [
            -16.3,
            13.8
          ],
          "per_pair_median_pct": 6.7,
          "share_of_pairs_lower": 0.444,
          "signflip_p": 0.972656,
          "signflip_method": "exact"
        },
        "2": {
          "pairs": 10,
          "control_sum": 3518636,
          "treatment_sum": 3274036,
          "pooled_change_pct": -7.0,
          "ci95_pct": [
            -19.4,
            4.6
          ],
          "per_pair_median_pct": -8.6,
          "share_of_pairs_lower": 0.6,
          "signflip_p": 0.294922,
          "signflip_method": "exact"
        },
        "3": {
          "pairs": 10,
          "control_sum": 3452200,
          "treatment_sum": 2928166,
          "pooled_change_pct": -15.2,
          "ci95_pct": [
            -24.6,
            -5.0
          ],
          "per_pair_median_pct": -7.5,
          "share_of_pairs_lower": 0.9,
          "signflip_p": 0.017578,
          "signflip_method": "exact"
        },
        "4": {
          "pairs": 10,
          "control_sum": 3551018,
          "treatment_sum": 3370287,
          "pooled_change_pct": -5.1,
          "ci95_pct": [
            -23.9,
            13.3
          ],
          "per_pair_median_pct": -2.3,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.814453,
          "signflip_method": "exact"
        },
        "5": {
          "pairs": 4,
          "control_sum": 2096329,
          "treatment_sum": 1673265,
          "pooled_change_pct": -20.2,
          "ci95_pct": [
            -49.1,
            19.0
          ],
          "per_pair_median_pct": -0.6,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.75,
          "signflip_method": "exact"
        }
      },
      "aa_gate": "passed",
      "version_0_1_4_on_vs_0_1_5_on": null
    }
  },
  "aa_all": {
    "pairs": 14,
    "control_sum": 4840713,
    "treatment_sum": 4787986,
    "pooled_change_pct": -1.1,
    "ci95_pct": [
      -8.4,
      5.7
    ],
    "per_pair_median_pct": -0.1,
    "share_of_pairs_lower": 0.5,
    "signflip_p": 0.88269,
    "signflip_method": "exact",
    "gate": "passed"
  },
  "usage": {
    "launch_gate_checks": 59,
    "weekly_used_pct_first_gate": 4.0,
    "weekly_used_pct_last_gate": 67.0,
    "weekly_used_pct_max_attempt_end": 68.0,
    "weekly_resets_at": 1794593260,
    "stop_reasons": []
  },
  "load": {
    "note": "the Claude heavy baseline ran on this machine concurrently; 1-minute load average",
    "attempts": 116,
    "start_median": 41.7,
    "start_p90": 110.9,
    "session_mean_median": 7.9,
    "session_mean_p90": 27.2,
    "session_max_max": 466.4
  },
  "ledger": {
    "note": "no token allowance (the weekly window governs); charged = reported + hidden upper bound",
    "attempts_charged": 118,
    "charged_non_cached": 47128909,
    "charged_cached": 2081422464,
    "unknown_usage_attempts": 3,
    "reported_non_cached": 40688927,
    "reported_cached": 1960621696,
    "hidden_upper_non_cached": 803896,
    "smoke_attempts": 0
  },
  "gates": [
    {
      "gate": "c6 A/A within 3 points",
      "outcome": "passed"
    },
    {
      "gate": "c6 quality non-inferior at -5 points",
      "outcome": "FAILED",
      "detail": {
        "graded_pairs": 43,
        "control_accepted": 38,
        "treatment_accepted": 37,
        "control_only": 1,
        "treatment_only": 0,
        "mcnemar_exact_p": 1.0,
        "pass_rate_diff_points": -2.3,
        "ci95_points": [
          -6.8,
          2.2
        ],
        "non_inferior_at_minus_5": false
      }
    },
    {
      "gate": "all A/A pairs within 3 points",
      "outcome": "passed"
    }
  ],
  "headline": {
    "c6_non_cached_pct": -8.7,
    "c6_ci95_pct": [
      -16.1,
      -1.3
    ],
    "c6_non_cached_pct_with_hidden_upper": -5.2,
    "c6_ci95_pct_with_hidden_upper": [
      -13.8,
      3.6
    ],
    "s1_non_cached_pct": null,
    "s1_ci95_pct": null,
    "publishable_as_headline": true,
    "blocked_by": []
  },
  "cache_sharing": {
    "aa_pairs": 14,
    "first_launched_mean_first_cache_read": 6674,
    "second_launched_mean_first_cache_read": 5394
  },
  "attempts_sha256": "c2a395e640f1777c60e231e721b3a5d1bbf48acda794472107d04b7c432f974a"
}