{
  "plan_id": "claude-pairs-analysis-2026-11-08-heavy-jail-059-v1",
  "run": "claude-code/2026-11-08-heavy-jail-059",
  "attempts": 82,
  "pairs": 41,
  "cost_usd_estimate_total": 372.74,
  "suites": {
    "s6": {
      "name": "S6 heavy long sessions (Django, SymPy at pinned commits; 14 or 15 scripted turns)",
      "pairs_run": 10,
      "valid_pairs": 10,
      "invalid": [],
      "primary_non_cached": {
        "pairs": 10,
        "control_sum": 5269907,
        "treatment_sum": 5161208,
        "pooled_change_pct": -2.1,
        "ci95_pct": [
          -11.1,
          7.8
        ],
        "per_pair_median_pct": -4.6,
        "share_of_pairs_lower": 0.6,
        "signflip_p": 0.685547,
        "signflip_method": "exact"
      },
      "secondary": {
        "total": {
          "pairs": 10,
          "control_sum": 205897461,
          "treatment_sum": 206526437,
          "pooled_change_pct": 0.3,
          "ci95_pct": [
            -14.6,
            16.9
          ],
          "per_pair_median_pct": -0.8,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.96875,
          "signflip_method": "exact"
        },
        "cached": {
          "pairs": 10,
          "control_sum": 200627554,
          "treatment_sum": 201365229,
          "pooled_change_pct": 0.4,
          "ci95_pct": [
            -14.7,
            17.2
          ],
          "per_pair_median_pct": -0.7,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.964844,
          "signflip_method": "exact"
        },
        "cost": {
          "pairs": 10,
          "control_sum": 94.1979748,
          "treatment_sum": 93.5305778,
          "pooled_change_pct": -0.7,
          "ci95_pct": [
            -11.5,
            11.0
          ],
          "per_pair_median_pct": -1.8,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.902344,
          "signflip_method": "exact"
        },
        "wall": {
          "pairs": 10,
          "control_sum": 12990.483,
          "treatment_sum": 13264.755,
          "pooled_change_pct": 2.1,
          "ci95_pct": [
            -6.1,
            12.2
          ],
          "per_pair_median_pct": 1.7,
          "share_of_pairs_lower": 0.4,
          "signflip_p": 0.666016,
          "signflip_method": "exact"
        },
        "turns": {
          "pairs": 10,
          "control_sum": 37,
          "treatment_sum": 34,
          "pooled_change_pct": -8.1,
          "ci95_pct": [
            -34.2,
            18.2
          ],
          "per_pair_median_pct": 0.0,
          "share_of_pairs_lower": 0.2,
          "signflip_p": 0.875,
          "signflip_method": "exact"
        }
      },
      "categories": {
        "input_tokens": {
          "pairs": 10,
          "control_sum": 2060,
          "treatment_sum": 2132,
          "pooled_change_pct": 3.5,
          "ci95_pct": [
            -5.3,
            13.4
          ],
          "per_pair_median_pct": 4.2,
          "share_of_pairs_lower": 0.4,
          "signflip_p": 0.511719,
          "signflip_method": "exact"
        },
        "cache_creation_input_tokens": {
          "pairs": 10,
          "control_sum": 4274393,
          "treatment_sum": 4161043,
          "pooled_change_pct": -2.7,
          "ci95_pct": [
            -12.6,
            8.5
          ],
          "per_pair_median_pct": -6.2,
          "share_of_pairs_lower": 0.6,
          "signflip_p": 0.642578,
          "signflip_method": "exact"
        },
        "cache_read_input_tokens": {
          "pairs": 10,
          "control_sum": 200627554,
          "treatment_sum": 201365229,
          "pooled_change_pct": 0.4,
          "ci95_pct": [
            -14.7,
            17.2
          ],
          "per_pair_median_pct": -0.7,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.964844,
          "signflip_method": "exact"
        },
        "output_tokens": {
          "pairs": 10,
          "control_sum": 993454,
          "treatment_sum": 998033,
          "pooled_change_pct": 0.5,
          "ci95_pct": [
            -7.6,
            9.6
          ],
          "per_pair_median_pct": 1.6,
          "share_of_pairs_lower": 0.4,
          "signflip_p": 0.916016,
          "signflip_method": "exact"
        }
      },
      "quality": {
        "graded_pairs": 10,
        "control_accepted": 9,
        "treatment_accepted": 9,
        "control_only": 0,
        "treatment_only": 0,
        "mcnemar_exact_p": 1.0,
        "pass_rate_diff_points": 0.0,
        "ci95_points": [
          0.0,
          0.0
        ],
        "non_inferior_at_minus_5": true
      },
      "pass_rate": {
        "off": {
          "accepted": 9,
          "graded": 10,
          "rate_pct": 90.0
        },
        "on": {
          "accepted": 9,
          "graded": 10,
          "rate_pct": 90.0
        }
      },
      "trims_on": 38,
      "trims_off": 0,
      "levers_on": {
        "trim": {
          "events": 28,
          "est_tokens_avoided": 46810
        },
        "read_guard": {
          "events": 10,
          "est_tokens_avoided": 103936
        }
      },
      "levers_on_sessions": {
        "trim": 9,
        "read_guard": 7
      },
      "on_sessions_lever_fired": 10,
      "on_sessions_inert": 0,
      "non_cached_fired_sessions": {
        "pairs": 10,
        "control_sum": 5269907,
        "treatment_sum": 5161208,
        "pooled_change_pct": -2.1,
        "ci95_pct": [
          -11.1,
          7.8
        ],
        "per_pair_median_pct": -4.6,
        "share_of_pairs_lower": 0.6,
        "signflip_p": 0.685547,
        "signflip_method": "exact"
      },
      "non_cached_inert_sessions": null,
      "aa": {
        "pairs": 10,
        "control_sum": 5541815,
        "treatment_sum": 4613484,
        "pooled_change_pct": -16.8,
        "ci95_pct": [
          -25.0,
          -9.7
        ],
        "per_pair_median_pct": -13.9,
        "share_of_pairs_lower": 0.9,
        "signflip_p": 0.003906,
        "signflip_method": "exact"
      },
      "aa_pairs": 10,
      "aa_pairs_run": 10,
      "aa_total": {
        "pairs": 10,
        "control_sum": 216014010,
        "treatment_sum": 170303074,
        "pooled_change_pct": -21.2,
        "ci95_pct": [
          -31.3,
          -8.8
        ],
        "per_pair_median_pct": -22.4,
        "share_of_pairs_lower": 0.8,
        "signflip_p": 0.023438,
        "signflip_method": "exact"
      },
      "aa_pairs_per_task": {
        "django-cache-headers": 1,
        "django-forms-fields": 1,
        "django-http-request": 1,
        "django-template-filters": 1,
        "django-urls-reverse": 1,
        "sympy-iterables": 1,
        "sympy-latex-printing": 1,
        "sympy-ntheory": 1,
        "sympy-permutations": 1,
        "sympy-sets-intervals": 1
      },
      "per_task": {
        "django-cache-headers": {
          "pairs": 1,
          "control_sum": 709299,
          "treatment_sum": 638840,
          "pooled_change_pct": -9.9,
          "ci95_pct": [
            -9.9,
            -9.9
          ],
          "per_pair_median_pct": -9.9,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 0,
              "graded": 1,
              "rate_pct": 0.0
            },
            "on": {
              "accepted": 0,
              "graded": 1,
              "rate_pct": 0.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "django-forms-fields": {
          "pairs": 1,
          "control_sum": 509397,
          "treatment_sum": 381706,
          "pooled_change_pct": -25.1,
          "ci95_pct": [
            -25.1,
            -25.1
          ],
          "per_pair_median_pct": -25.1,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "django-http-request": {
          "pairs": 1,
          "control_sum": 423289,
          "treatment_sum": 488820,
          "pooled_change_pct": 15.5,
          "ci95_pct": [
            15.5,
            15.5
          ],
          "per_pair_median_pct": 15.5,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "django-template-filters": {
          "pairs": 1,
          "control_sum": 315221,
          "treatment_sum": 403186,
          "pooled_change_pct": 27.9,
          "ci95_pct": [
            27.9,
            27.9
          ],
          "per_pair_median_pct": 27.9,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "django-urls-reverse": {
          "pairs": 1,
          "control_sum": 413743,
          "treatment_sum": 311704,
          "pooled_change_pct": -24.7,
          "ci95_pct": [
            -24.7,
            -24.7
          ],
          "per_pair_median_pct": -24.7,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "sympy-iterables": {
          "pairs": 1,
          "control_sum": 569913,
          "treatment_sum": 522233,
          "pooled_change_pct": -8.4,
          "ci95_pct": [
            -8.4,
            -8.4
          ],
          "per_pair_median_pct": -8.4,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "sympy-latex-printing": {
          "pairs": 1,
          "control_sum": 597074,
          "treatment_sum": 656263,
          "pooled_change_pct": 9.9,
          "ci95_pct": [
            9.9,
            9.9
          ],
          "per_pair_median_pct": 9.9,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "sympy-ntheory": {
          "pairs": 1,
          "control_sum": 505287,
          "treatment_sum": 606121,
          "pooled_change_pct": 20.0,
          "ci95_pct": [
            20.0,
            20.0
          ],
          "per_pair_median_pct": 20.0,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "sympy-permutations": {
          "pairs": 1,
          "control_sum": 651589,
          "treatment_sum": 581911,
          "pooled_change_pct": -10.7,
          "ci95_pct": [
            -10.7,
            -10.7
          ],
          "per_pair_median_pct": -10.7,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "sympy-sets-intervals": {
          "pairs": 1,
          "control_sum": 575095,
          "treatment_sum": 570424,
          "pooled_change_pct": -0.8,
          "ci95_pct": [
            -0.8,
            -0.8
          ],
          "per_pair_median_pct": -0.8,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        }
      },
      "rounds": {
        "1": {
          "pairs": 10,
          "control_sum": 5269907,
          "treatment_sum": 5161208,
          "pooled_change_pct": -2.1,
          "ci95_pct": [
            -11.1,
            7.8
          ],
          "per_pair_median_pct": -4.6,
          "share_of_pairs_lower": 0.6,
          "signflip_p": 0.685547,
          "signflip_method": "exact"
        }
      },
      "per_attempt": {
        "off": {
          "cost_usd_estimate_mean": 9.42,
          "wall_seconds_mean": 1299.048,
          "non_cached_mean": 526990.7,
          "total_mean": 20589746.1
        },
        "on": {
          "cost_usd_estimate_mean": 9.353,
          "wall_seconds_mean": 1326.476,
          "non_cached_mean": 516120.8,
          "total_mean": 20652643.7
        }
      },
      "load_average_1m_start": {
        "mean": 98.648,
        "max": 325.02
      },
      "aa_gate": "flagged: A/A CI [-25.0, -9.7] excludes 0 by more than 3 points",
      "groups": {
        "s6": {
          "pairs": 10,
          "primary_non_cached": {
            "pairs": 10,
            "control_sum": 5269907,
            "treatment_sum": 5161208,
            "pooled_change_pct": -2.1,
            "ci95_pct": [
              -11.1,
              7.8
            ],
            "per_pair_median_pct": -4.6,
            "share_of_pairs_lower": 0.6,
            "signflip_p": 0.685547,
            "signflip_method": "exact"
          },
          "quality": {
            "graded_pairs": 10,
            "control_accepted": 9,
            "treatment_accepted": 9,
            "control_only": 0,
            "treatment_only": 0,
            "mcnemar_exact_p": 1.0,
            "pass_rate_diff_points": 0.0,
            "ci95_points": [
              0.0,
              0.0
            ],
            "non_inferior_at_minus_5": true
          }
        }
      }
    },
    "s1": {
      "name": "S1 heavy-output (frozen 2026-10-22 task set), effort medium",
      "pairs_run": 16,
      "valid_pairs": 16,
      "invalid": [],
      "primary_non_cached": {
        "pairs": 16,
        "control_sum": 206604,
        "treatment_sum": 150853,
        "pooled_change_pct": -27.0,
        "ci95_pct": [
          -40.1,
          -9.8
        ],
        "per_pair_median_pct": -1.4,
        "share_of_pairs_lower": 0.562,
        "signflip_p": 0.018299,
        "signflip_method": "monte-carlo"
      },
      "secondary": {
        "total": {
          "pairs": 16,
          "control_sum": 1214619,
          "treatment_sum": 1080056,
          "pooled_change_pct": -11.1,
          "ci95_pct": [
            -22.9,
            1.6
          ],
          "per_pair_median_pct": -0.1,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.136893,
          "signflip_method": "monte-carlo"
        },
        "cached": {
          "pairs": 16,
          "control_sum": 1008015,
          "treatment_sum": 929203,
          "pooled_change_pct": -7.8,
          "ci95_pct": [
            -20.7,
            6.0
          ],
          "per_pair_median_pct": -0.0,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.310334,
          "signflip_method": "monte-carlo"
        },
        "cost": {
          "pairs": 16,
          "control_sum": 2.075103,
          "treatment_sum": 1.6243566,
          "pooled_change_pct": -21.7,
          "ci95_pct": [
            -33.7,
            -7.3
          ],
          "per_pair_median_pct": -3.3,
          "share_of_pairs_lower": 0.625,
          "signflip_p": 0.019149,
          "signflip_method": "monte-carlo"
        },
        "wall": {
          "pairs": 16,
          "control_sum": 287.217,
          "treatment_sum": 303.734,
          "pooled_change_pct": 5.8,
          "ci95_pct": [
            -4.6,
            17.0
          ],
          "per_pair_median_pct": -0.0,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.347733,
          "signflip_method": "monte-carlo"
        },
        "turns": {
          "pairs": 16,
          "control_sum": 91,
          "treatment_sum": 90,
          "pooled_change_pct": -1.1,
          "ci95_pct": [
            -13.8,
            17.7
          ],
          "per_pair_median_pct": 0.0,
          "share_of_pairs_lower": 0.312,
          "signflip_p": 1.0,
          "signflip_method": "monte-carlo"
        }
      },
      "categories": {
        "input_tokens": {
          "pairs": 16,
          "control_sum": 144,
          "treatment_sum": 148,
          "pooled_change_pct": 2.8,
          "ci95_pct": [
            -9.1,
            16.7
          ],
          "per_pair_median_pct": 0.0,
          "share_of_pairs_lower": 0.25,
          "signflip_p": 0.847358,
          "signflip_method": "monte-carlo"
        },
        "cache_creation_input_tokens": {
          "pairs": 16,
          "control_sum": 188023,
          "treatment_sum": 131348,
          "pooled_change_pct": -30.1,
          "ci95_pct": [
            -43.8,
            -11.3
          ],
          "per_pair_median_pct": -0.9,
          "share_of_pairs_lower": 0.562,
          "signflip_p": 0.015449,
          "signflip_method": "monte-carlo"
        },
        "cache_read_input_tokens": {
          "pairs": 16,
          "control_sum": 1008015,
          "treatment_sum": 929203,
          "pooled_change_pct": -7.8,
          "ci95_pct": [
            -20.7,
            6.0
          ],
          "per_pair_median_pct": -0.0,
          "share_of_pairs_lower": 0.5,
          "signflip_p": 0.310334,
          "signflip_method": "monte-carlo"
        },
        "output_tokens": {
          "pairs": 16,
          "control_sum": 18437,
          "treatment_sum": 19357,
          "pooled_change_pct": 5.0,
          "ci95_pct": [
            -5.8,
            17.3
          ],
          "per_pair_median_pct": 3.0,
          "share_of_pairs_lower": 0.375,
          "signflip_p": 0.432478,
          "signflip_method": "monte-carlo"
        }
      },
      "quality": {
        "graded_pairs": 16,
        "control_accepted": 16,
        "treatment_accepted": 16,
        "control_only": 0,
        "treatment_only": 0,
        "mcnemar_exact_p": 1.0,
        "pass_rate_diff_points": 0.0,
        "ci95_points": [
          0.0,
          0.0
        ],
        "non_inferior_at_minus_5": true
      },
      "pass_rate": {
        "off": {
          "accepted": 16,
          "graded": 16,
          "rate_pct": 100.0
        },
        "on": {
          "accepted": 16,
          "graded": 16,
          "rate_pct": 100.0
        }
      },
      "trims_on": 3,
      "trims_off": 0,
      "levers_on": {
        "trim": {
          "events": 2,
          "est_tokens_avoided": 6596
        },
        "read_guard": {
          "events": 1,
          "est_tokens_avoided": 12013
        }
      },
      "levers_on_sessions": {
        "trim": 2,
        "read_guard": 1
      },
      "on_sessions_lever_fired": 3,
      "on_sessions_inert": 13,
      "non_cached_fired_sessions": {
        "pairs": 3,
        "control_sum": 80245,
        "treatment_sum": 43114,
        "pooled_change_pct": -46.3,
        "ci95_pct": [
          -61.3,
          -23.7
        ],
        "per_pair_median_pct": -45.6,
        "share_of_pairs_lower": 1.0,
        "signflip_p": 0.25,
        "signflip_method": "exact"
      },
      "non_cached_inert_sessions": {
        "pairs": 13,
        "control_sum": 126359,
        "treatment_sum": 107739,
        "pooled_change_pct": -14.7,
        "ci95_pct": [
          -31.4,
          0.2
        ],
        "per_pair_median_pct": 0.0,
        "share_of_pairs_lower": 0.462,
        "signflip_p": 0.157715,
        "signflip_method": "exact"
      },
      "aa": {
        "pairs": 5,
        "control_sum": 81126,
        "treatment_sum": 79387,
        "pooled_change_pct": -2.1,
        "ci95_pct": [
          -4.2,
          -0.2
        ],
        "per_pair_median_pct": -1.6,
        "share_of_pairs_lower": 0.8,
        "signflip_p": 0.125,
        "signflip_method": "exact"
      },
      "aa_pairs": 5,
      "aa_pairs_run": 5,
      "aa_total": {
        "pairs": 5,
        "control_sum": 532624,
        "treatment_sum": 470287,
        "pooled_change_pct": -11.7,
        "ci95_pct": [
          -24.9,
          0.5
        ],
        "per_pair_median_pct": -0.2,
        "share_of_pairs_lower": 0.6,
        "signflip_p": 0.375,
        "signflip_method": "exact"
      },
      "aa_pairs_per_task": {
        "contract-resync": 1,
        "iterative-check-report": 1,
        "large-source": 1,
        "large-source-extended": 1,
        "real-harness-source": 1
      },
      "per_task": {
        "artifact-recovery": {
          "pairs": 1,
          "control_sum": 13298,
          "treatment_sum": 5666,
          "pooled_change_pct": -57.4,
          "ci95_pct": [
            -57.4,
            -57.4
          ],
          "per_pair_median_pct": -57.4,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "ci-log-triage": {
          "pairs": 1,
          "control_sum": 8309,
          "treatment_sum": 7767,
          "pooled_change_pct": -6.5,
          "ci95_pct": [
            -6.5,
            -6.5
          ],
          "per_pair_median_pct": -6.5,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "config-lookup": {
          "pairs": 1,
          "control_sum": 6450,
          "treatment_sum": 6529,
          "pooled_change_pct": 1.2,
          "ci95_pct": [
            1.2,
            1.2
          ],
          "per_pair_median_pct": 1.2,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "contract-resync": {
          "pairs": 1,
          "control_sum": 22593,
          "treatment_sum": 21354,
          "pooled_change_pct": -5.5,
          "ci95_pct": [
            -5.5,
            -5.5
          ],
          "per_pair_median_pct": -5.5,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "cross-file": {
          "pairs": 1,
          "control_sum": 5535,
          "treatment_sum": 6023,
          "pooled_change_pct": 8.8,
          "ci95_pct": [
            8.8,
            8.8
          ],
          "per_pair_median_pct": 8.8,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "iterative-check": {
          "pairs": 1,
          "control_sum": 16750,
          "treatment_sum": 12788,
          "pooled_change_pct": -23.7,
          "ci95_pct": [
            -23.7,
            -23.7
          ],
          "per_pair_median_pct": -23.7,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "iterative-check-report": {
          "pairs": 1,
          "control_sum": 26820,
          "treatment_sum": 10375,
          "pooled_change_pct": -61.3,
          "ci95_pct": [
            -61.3,
            -61.3
          ],
          "per_pair_median_pct": -61.3,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "large-source": {
          "pairs": 1,
          "control_sum": 16246,
          "treatment_sum": 6103,
          "pooled_change_pct": -62.4,
          "ci95_pct": [
            -62.4,
            -62.4
          ],
          "per_pair_median_pct": -62.4,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "large-source-extended": {
          "pairs": 1,
          "control_sum": 5966,
          "treatment_sum": 6086,
          "pooled_change_pct": 2.0,
          "ci95_pct": [
            2.0,
            2.0
          ],
          "per_pair_median_pct": 2.0,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "medium-source": {
          "pairs": 1,
          "control_sum": 15058,
          "treatment_sum": 14888,
          "pooled_change_pct": -1.1,
          "ci95_pct": [
            -1.1,
            -1.1
          ],
          "per_pair_median_pct": -1.1,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "noisy-tests": {
          "pairs": 1,
          "control_sum": 6346,
          "treatment_sum": 6245,
          "pooled_change_pct": -1.6,
          "ci95_pct": [
            -1.6,
            -1.6
          ],
          "per_pair_median_pct": -1.6,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "noisy-tests-large": {
          "pairs": 1,
          "control_sum": 8769,
          "treatment_sum": 9239,
          "pooled_change_pct": 5.4,
          "ci95_pct": [
            5.4,
            5.4
          ],
          "per_pair_median_pct": 5.4,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "real-harness-source": {
          "pairs": 1,
          "control_sum": 36675,
          "treatment_sum": 19951,
          "pooled_change_pct": -45.6,
          "ci95_pct": [
            -45.6,
            -45.6
          ],
          "per_pair_median_pct": -45.6,
          "share_of_pairs_lower": 1.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 1
        },
        "stale-build": {
          "pairs": 1,
          "control_sum": 5433,
          "treatment_sum": 5465,
          "pooled_change_pct": 0.6,
          "ci95_pct": [
            0.6,
            0.6
          ],
          "per_pair_median_pct": 0.6,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "stale-source": {
          "pairs": 1,
          "control_sum": 5124,
          "treatment_sum": 5142,
          "pooled_change_pct": 0.4,
          "ci95_pct": [
            0.4,
            0.4
          ],
          "per_pair_median_pct": 0.4,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        },
        "verbose-suite": {
          "pairs": 1,
          "control_sum": 7232,
          "treatment_sum": 7232,
          "pooled_change_pct": 0.0,
          "ci95_pct": [
            0.0,
            0.0
          ],
          "per_pair_median_pct": 0.0,
          "share_of_pairs_lower": 0.0,
          "signflip_p": 1.0,
          "signflip_method": "exact",
          "pass_rate": {
            "off": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            },
            "on": {
              "accepted": 1,
              "graded": 1,
              "rate_pct": 100.0
            }
          },
          "on_sessions_lever_fired": 0
        }
      },
      "rounds": {
        "1": {
          "pairs": 16,
          "control_sum": 206604,
          "treatment_sum": 150853,
          "pooled_change_pct": -27.0,
          "ci95_pct": [
            -40.1,
            -9.8
          ],
          "per_pair_median_pct": -1.4,
          "share_of_pairs_lower": 0.562,
          "signflip_p": 0.018299,
          "signflip_method": "monte-carlo"
        }
      },
      "per_attempt": {
        "off": {
          "cost_usd_estimate_mean": 0.13,
          "wall_seconds_mean": 17.951,
          "non_cached_mean": 12912.75,
          "total_mean": 75913.688
        },
        "on": {
          "cost_usd_estimate_mean": 0.102,
          "wall_seconds_mean": 18.983,
          "non_cached_mean": 9428.312,
          "total_mean": 67503.5
        }
      },
      "load_average_1m_start": {
        "mean": 46.162,
        "max": 115.7
      },
      "aa_gate": "passed",
      "groups": {
        "s1-control": {
          "pairs": 10,
          "primary_non_cached": {
            "pairs": 10,
            "control_sum": 81554,
            "treatment_sum": 74196,
            "pooled_change_pct": -9.0,
            "ci95_pct": [
              -26.3,
              2.4
            ],
            "per_pair_median_pct": 0.2,
            "share_of_pairs_lower": 0.4,
            "signflip_p": 0.570312,
            "signflip_method": "exact"
          },
          "quality": {
            "graded_pairs": 10,
            "control_accepted": 10,
            "treatment_accepted": 10,
            "control_only": 0,
            "treatment_only": 0,
            "mcnemar_exact_p": 1.0,
            "pass_rate_diff_points": 0.0,
            "ci95_points": [
              0.0,
              0.0
            ],
            "non_inferior_at_minus_5": true
          }
        },
        "s1-lever": {
          "pairs": 6,
          "primary_non_cached": {
            "pairs": 6,
            "control_sum": 125050,
            "treatment_sum": 76657,
            "pooled_change_pct": -38.7,
            "ci95_pct": [
              -54.2,
              -17.4
            ],
            "per_pair_median_pct": -34.6,
            "share_of_pairs_lower": 0.833,
            "signflip_p": 0.0625,
            "signflip_method": "exact"
          },
          "quality": {
            "graded_pairs": 6,
            "control_accepted": 6,
            "treatment_accepted": 6,
            "control_only": 0,
            "treatment_only": 0,
            "mcnemar_exact_p": 1.0,
            "pass_rate_diff_points": 0.0,
            "ci95_points": [
              0.0,
              0.0
            ],
            "non_inferior_at_minus_5": true
          }
        }
      }
    }
  },
  "window": {
    "first": {
      "status": "allowed",
      "five_hour_utilization": null,
      "five_hour_resetsAt": null,
      "seven_day_utilization": 0.72,
      "seven_day_resetsAt": 1794326400
    },
    "last": {
      "status": "allowed_warning",
      "five_hour_utilization": null,
      "five_hour_resetsAt": null,
      "seven_day_utilization": 0.88,
      "seven_day_resetsAt": 1794326400
    },
    "readings": 111
  },
  "cache_sharing": {
    "aa_pairs": 15,
    "first_launched_mean_first_cache_read": 7002,
    "second_launched_mean_first_cache_read": 7002
  },
  "attempts_sha256": "3e8c76d2590b7d19ed9895ea2f08c43f7cfb6d7b8080a77cb714d1d20cfbd637"
}