peeksafe 0.0.0-stage → 0.1.0

package/dist/cluster.d.ts

@@ -0,0 +1,328 @@
+/**
+ * cluster.ts: the suite-level effect, with the cases treated as what they are.
+ *
+ * ## The bug this module exists to fix
+ *
+ * The obvious summary of a suite effect is an ordinary normal interval over the
+ * per-case differences:
+ *
+ *     mean ± 1.96 · sd / √n         with n = 200 cases
+ *
+ * That formula is only correct if the 200 numbers are 200 independent draws.
+ * They usually are not. A suite of **40 case files, each expanded into 5
+ * variants**, is typical: `extract.pricing#saas` and `extract.pricing#api`
+ * share a prompt template, a grader, an expectation shape and a failure mode.
+ * When one moves, its four siblings move with it. The effective number of
+ * independent observations is closer to 40 than to 200, the true standard error
+ * is larger than √(s²/200), and the reported interval is **too narrow**.
+ *
+ * A too-narrow interval on the headline number is the exact failure this whole
+ * package exists to prevent, so it gets a proper estimator rather than a fudge
+ * factor.
+ *
+ * ## What is estimated here, and what is not
+ *
+ * Clustering touches **only the suite-level (and family-level) summary**.
+ * Specifically:
+ *
+ *   affected      the across-case mean difference and its interval
+ *                 (`clusteredEffect`), plus any family-level roll-up.
+ *
+ *   NOT affected  every per-case quantity. Each case's e-value is its own
+ *                 self-contained test on that case's own runs; nothing about
+ *                 case *A* enters case *B*'s likelihood ratio. And the
+ *                 multiplicity correction is **e-BH**, which controls FDR under
+ *                 *arbitrary* dependence between the tested hypotheses, that
+ *                 is the whole reason it was chosen over BH. So `gate`'s
+ *                 verdict, its regressed cases and its thresholds are all
+ *                 bit-for-bit identical whether or not the cases cluster.
+ *
+ * In other words: clustering makes the *description* honest. It does not, and
+ * must not, move the *decision*.
+ *
+ * ## Two estimators, because they fail in different directions
+ *
+ * **CR2 (the headline).** A cluster-robust sandwich with the Bell-McCaffrey
+ * bias reduction. For the intercept-only model that a mean is, the hat matrix
+ * is H = J/n, so each cluster's leverage block is (I − J_{n_g}/n) and the CR2
+ * adjustment `A_g = (I − H_gg)^{-1/2}` collapses to a scalar on the cluster's
+ * residual sum:
+ *
+ *     Var̂(ȳ) = (1/n²) · Σ_g  (Σ_{i∈g} ê_i)²  /  (1 − n_g/n)
+ *
+ * Assumption-light: it needs the *families* to be independent of each other and
+ * nothing at all about the correlation structure *inside* a family. It does not
+ * assume the family effects are normal, or exchangeable, or even
+ * identically distributed. Paired with Bell-McCaffrey/Satterthwaite degrees of
+ * freedom (which, for equal-sized families, reduce exactly to G − 1; see the
+ * derivation in the comment above `dfRaw` below) it is the estimator you want
+ * when there are 40 clusters rather than 4,000.
+ *
+ * **Random effects (the cross-check).** The one-way model
+ * y_gi = μ + u_g + e_gi with the ANOVA (Swamy-Arora) estimators for τ² and σ².
+ * More efficient *if* its assumptions hold, and it yields the number that
+ * actually explains the width change: the intra-class correlation
+ * ρ = τ²/(τ² + σ²). If the two estimators disagree materially, that is itself
+ * information, and `compareEstimators` says so rather than picking a winner.
+ *
+ * ## The design effect
+ *
+ * `designEffect` = (clustered SE / naive iid SE)². It is the factor by which
+ * the naive analysis overstated its own information, "these 200 cases carry
+ * the information of 200/DEFF independent ones". For roughly equal families of
+ * size k it tracks the textbook 1 + (k − 1)ρ, which is the sanity check the
+ * tests assert.
+ */
+import { type Interval } from "./stats.js";
+/**
+ * The cluster a case belongs to: its **case file**.
+ *
+ * Variant ids are expected as `<fileId>#<variant>`, so everything before
+ * the first `#` is the file, and the file is the unit an author actually wrote.
+ * A case with no `#` is its own family of one, which is correct, not a
+ * degenerate case: an unexpanded case file shares its template with nobody.
+ */
+export declare function caseFamily(caseId: string): string;
+/**
+ * The cluster key for a case: the one it **declares**, or its case file.
+ *
+ * `caseFamily` sees exactly one reason two cases correlate, that they were
+ * minted from the same file, and is blind to every other: a shared grader, a
+ * shared fixture directory, one flaky tool that six unrelated cases happen to
+ * call. Those cases are correlated too, and the interval will be too narrow
+ * again for a reason the id cannot express. So a case may declare its own key.
+ *
+ * A wrong key is worse than no key, which is why `clusterKeyDiagnostic` ships
+ * alongside: declaring two genuinely independent cases into one family throws
+ * away real information and widens the interval for nothing, and the only way
+ * to know is to look at the ICC the declared grouping actually exhibits.
+ */
+export declare const clusterKey: (c: {
+    id: string;
+    cluster?: string;
+}) => string;
+export interface ClusterObservation {
+    /** the family this observation belongs to */
+    cluster: string;
+    value: number;
+}
+/** Group observations by cluster, preserving first-seen order. */
+export declare function groupByCluster(obs: ClusterObservation[]): Map<string, number[]>;
+export interface ClusteredMean extends Interval {
+    /** the mean itself, same number the naive estimator reports */
+    point: number;
+    low: number;
+    high: number;
+    /** number of *cases* behind the estimate */
+    observations: number;
+    /** number of *independent* units behind the estimate, the one that matters */
+    clusters: number;
+    largestCluster: number;
+    meanClusterSize: number;
+    /** CR2 cluster-robust standard error of the mean */
+    se: number;
+    /** the iid standard error the naive interval used, s/√n */
+    naiveSe: number;
+    /** (se / naiveSe)², the factor by which the naive analysis overstated its information */
+    designEffect: number;
+    /** how much wider the honest interval is, half-width to half-width */
+    widthRatio: number;
+    /** Bell-McCaffrey / Satterthwaite degrees of freedom */
+    df: number;
+    /** the t multiplier actually used (vs `normalQuantile` for the naive one) */
+    tCritical: number;
+    /** how many independent cases this suite is *worth*, n / designEffect */
+    effectiveSampleSize: number;
+    method: "CR2";
+    /** set when the estimate could not be formed; `low`/`high` are then vacuous */
+    degenerate: string | null;
+}
+/** The naive iid interval, kept so the two can be printed side by side. */
+export declare function iidMean(values: number[], level?: number): Interval;
+/**
+ * CR2 cluster-robust interval for a mean, with Bell-McCaffrey degrees of
+ * freedom. See the module header for the derivation; the short version is that
+ * for an intercept-only design the whole apparatus collapses to
+ *
+ *     Var̂ = (1/n²) Σ_g (Σ_{i∈g} ê_i)² / (1 − n_g/n)
+ *
+ * and the Satterthwaite df to a closed form in the cluster sizes alone.
+ */
+export declare function clusterRobustMean(obs: ClusterObservation[], level?: number): ClusteredMean;
+export interface RandomEffectsMean extends Interval {
+    clusters: number;
+    observations: number;
+    /** between-family variance τ̂² (ANOVA / Swamy-Arora, floored at 0) */
+    tauSquared: number;
+    /** within-family variance σ̂² */
+    sigmaSquared: number;
+    /** intra-class correlation τ̂²/(τ̂²+σ̂²), the number that explains the widening */
+    icc: number;
+    se: number;
+    df: number;
+    method: "random-effects";
+    degenerate: string | null;
+}
+/**
+ * One-way random-effects (hierarchical) estimate of the same mean.
+ *
+ *   y_gi = μ + u_g + e_gi,   u_g ~ (0, τ²),   e_gi ~ (0, σ²)
+ *
+ * τ² and σ² come from the ANOVA decomposition (the Swamy-Arora estimator,
+ * unbiased and closed-form, and it does not need the normality that REML would
+ * lean on). μ̂ is the GLS weighted mean of the family means with
+ * w_g = 1/(τ² + σ²/n_g), so its variance is 1/Σw_g.
+ *
+ * Two properties worth knowing before trusting it:
+ *  - at τ̂² = 0 it degenerates **exactly** to the naive iid interval, which is
+ *    the correct behaviour: no between-family variance means the cases really
+ *    were exchangeable;
+ *  - it weights families by information rather than by size, so unlike the
+ *    unweighted mean it is not the same point estimate as CR2 when families
+ *    differ in size. That difference is a feature and it is reported.
+ */
+export declare function randomEffectsMean(obs: ClusterObservation[], level?: number): RandomEffectsMean;
+export interface FamilyEffect {
+    family: string;
+    cases: number;
+    /** unweighted mean of the family's per-case effects */
+    point: number;
+    /**
+     * Spread of the family's own cases. Deliberately **not** an interval: the
+     * variants inside a family are the correlated things, so an interval over
+     * five of them would repeat the very mistake this module fixes, one level
+     * down. A family of five variants is one observation, not five.
+     */
+    sd: number;
+    min: number;
+    max: number;
+}
+export interface ClusteredEffect {
+    /** the honest headline: CR2 with small-cluster degrees of freedom */
+    cr2: ClusteredMean;
+    /** the hierarchical cross-check, and where the ICC comes from */
+    randomEffects: RandomEffectsMean;
+    /** the naive iid interval, which assumes every case is independent */
+    naive: Interval;
+    /** (clustered SE / naive SE)² */
+    designEffect: number;
+    /** how much wider the honest interval is */
+    widthRatio: number;
+    icc: number;
+    clusters: number;
+    observations: number;
+    /** per case-file roll-up, point estimates only, and the docs say why */
+    families: FamilyEffect[];
+    /**
+     * True when the naive interval excluded zero and the honest one does not.
+     * This is the finding: a claim that "the suite moved" which does not survive
+     * being told that the cases are not independent.
+     */
+    signFlippedByClustering: boolean;
+    /** one line an operator can read without knowing what CR2 is */
+    headline: string;
+}
+/**
+ * The whole suite-level story in one object: the honest interval, the naive one
+ * it replaces, the factor between them, and whether swapping them changes what
+ * a reader would conclude.
+ */
+export declare function clusteredEffect(obs: ClusterObservation[], level?: number): ClusteredEffect;
+/**
+ * Do the two estimators tell the same story? They rest on different
+ * assumptions, so agreement is evidence and disagreement is a warning worth
+ * printing rather than resolving by fiat.
+ */
+export declare function compareEstimators(e: ClusteredEffect): {
+    agree: boolean;
+    /** ratio of the two standard errors, robust ÷ model-based */
+    seRatio: number;
+    note: string;
+};
+export interface GroupingSummary {
+    key: string;
+    /** why this grouping has no estimable interval, when it has none */
+    degenerate: string | null;
+    clusters: number;
+    meanClusterSize: number;
+    /**
+     * The ANOVA intra-class correlation, **only when `iccEstimable`**. With one
+     * case per family σ̂² is 0 by construction and this comes out at exactly 1,
+     * which is an artifact of the decomposition and not a measurement: a grouping
+     * of singletons says nothing at all about within-family correlation. Read the
+     * flag before the number.
+     */
+    icc: number;
+    designEffect: number;
+    /** how much wider this grouping's interval is than the iid one. Not 1/DEFF:
+     *  a coarse grouping also costs degrees of freedom, and a few large clusters
+     *  can have a *smaller* design effect and a *much wider* interval. */
+    widthRatio: number;
+    se: number;
+    /** cases per independent unit, `observations / designEffect` */
+    effectiveSampleSize: number;
+    /** false when the random-effects fit cannot separate τ² from σ², so `icc` is not a measurement */
+    iccEstimable: boolean;
+}
+export interface ClusterKeyDiagnostic {
+    declared: GroupingSummary;
+    /** the same observations grouped by case file, which is what ships by default */
+    fallback: GroupingSummary;
+    /**
+     * The declared grouping is *coarser* than the fallback and its groups show no
+     * material correlation, it merged cases that do not move together, and got
+     * nothing for it.
+     */
+    mergesUncorrelatedCases: boolean;
+    /**
+     * …and the resulting interval is **narrower** than the default's, on too few
+     * clusters for CR2 to be trusted. This is the dangerous direction, and it was
+     * not the one the first version of this diagnostic looked for: a coarse key
+     * with near-zero ICC can have a design effect *below* 1 and produce a tighter
+     * interval than the honest one, which is the exact failure `cluster.ts` was
+     * written to remove, reintroduced by a user-supplied key.
+     */
+    narrowsOnTooFewClusters: boolean;
+    /**
+     * The mirror image, and the one the first two flags cannot see because both
+     * require the declared key to be *coarser*: a key that is **finer** than the
+     * case-file default, splitting apart families that do move together, and
+     * buying a narrower interval with the split. It is the same too-narrow
+     * headline `narrowsOnTooFewClusters` catches, arrived at from the other side,
+     * and it is not a small effect: dissolving eight ten-variant files into
+     * eighty singletons takes the interval from 3.90× the iid one to 1.02×, a
+     * claim that the suite carries eighty independent cases' worth of evidence
+     * when it carries about eight. Both figures are asserted in
+     * `cluster.test.ts`.
+     */
+    narrowsBySplitting: boolean;
+    /** True when the declared grouping found correlation the case ids could not see. */
+    findsHiddenCorrelation: boolean;
+    verdict: string;
+}
+/**
+ * Below this many clusters, a cluster-robust standard error is optimistic even
+ * with small-cluster degrees of freedom. Standard practice; `compareEstimators`
+ * says the same thing from the other direction.
+ */
+export declare const MIN_TRUSTWORTHY_CLUSTERS = 20;
+/**
+ * Does a user-declared cluster key carry anything?
+ *
+ * The honest answer is a comparison, not a number: the same observations are
+ * grouped both ways and the intra-class correlation of each grouping is
+ * reported. A declared key that merges cases which do not in fact move together
+ * shows an ICC near zero while having *fewer, larger* clusters than the
+ * fallback, it has thrown away real information and widened the interval to
+ * pay for it. A declared key that finds correlation the case ids could not see
+ * shows a higher ICC and a larger design effect, and is doing its job. A
+ * declared key that is *finer* than the fallback, splitting apart files whose
+ * variants do move together, narrows the interval instead, and that is the
+ * dangerous direction rather than the merely wasteful one.
+ *
+ * Neither outcome is corrected automatically. A cluster key is a claim about
+ * the world and only the person who made it can say whether it is true; this
+ * says what the data thinks of it.
+ */
+export declare function clusterKeyDiagnostic(declared: ClusterObservation[], fallback: ClusterObservation[]): ClusterKeyDiagnostic;
+//# sourceMappingURL=cluster.d.ts.map

package/dist/cluster.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster.d.ts","sourceRoot":"","sources":["../src/cluster.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0EG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAA6B,MAAM,YAAY,CAAC;AAItE;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAGjD;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,UAAU,MAAO;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,KAAG,MAKhE,CAAC;AAEF,MAAM,WAAW,kBAAkB;IACjC,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAgBD,kEAAkE;AAClE,wBAAgB,cAAc,CAAC,GAAG,EAAE,kBAAkB,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAiB/E;AAID,MAAM,WAAW,aAAc,SAAQ,QAAQ;IAC7C,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,oDAAoD;IACpD,EAAE,EAAE,MAAM,CAAC;IACX,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,EAAE,EAAE,MAAM,CAAC;IACX,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,MAAM,EAAE,KAAK,CAAC;IACd,+EAA+E;IAC/E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,2EAA2E;AAC3E,wBAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,SAAO,GAAG,QAAQ,CAShE;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE,KAAK,SAAO,GAAG,aAAa,CAkGxF;AAID,MAAM,WAAW,iBAAkB,SAAQ,QAAQ;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,mFAAmF;IACnF,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,gBAAgB,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE,KAAK,SAAO,GAAG,iBAAiB,CA0H5F;AAID,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,GAAG,EAAE,aAAa,CAAC;IACnB,iEAAiE;IACjE,aAAa,EAAE,iBAAiB,CAAC;IACjC,sEAAsE;IACtE,KAAK,EAAE,QAAQ,CAAC;IAChB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB;;;;OAIG;IACH,uBAAuB,EAAE,OAAO,CAAC;IACjC,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE,KAAK,SAAO,GAAG,eAAe,CA4CxF;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,eAAe,GAAG;IACrD,KAAK,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAyBA;AAID,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,oEAAoE;IACpE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB;;0EAEsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,gEAAgE;IAChE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kGAAkG;IAClG,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,eAAe,CAAC;IAC1B,iFAAiF;IACjF,QAAQ,EAAE,eAAe,CAAC;IAC1B;;;;OAIG;IACH,uBAAuB,EAAE,OAAO,CAAC;IACjC;;;;;;;OAOG;IACH,uBAAuB,EAAE,OAAO,CAAC;IACjC;;;;;;;;;;;OAWG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B,oFAAoF;IACpF,sBAAsB,EAAE,OAAO,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,KAAK,CAAC;AA4B3C;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,kBAAkB,EAAE,EAC9B,QAAQ,EAAE,kBAAkB,EAAE,GAC7B,oBAAoB,CA6GtB"}