quantred 2.4.0__py3-none-any.whl

quantred/analytics/screening.py

@@ -0,0 +1,190 @@
+"""Reading a parameter sweep without fooling yourself with the best point.
+
+A screen evaluates many configurations and the natural thing to print is the winner.
+That is the one number a sweep cannot support: **the best of N is the maximum of N
+draws**, and the maximum of N draws from pure noise is a positive number that grows
+with N. A leaderboard with no null beside it is a selection-bias machine — the same
+error the Deflated Sharpe exists to prevent, one layer up, with no deflation applied.
+
+So the summary leads with the *distribution* — how many points, where the middle sits,
+how wide the spread is, how many were positive at all — and any statement about a best
+point carries what the best of that many draws is worth under the null.
+
+The second thing worth reading off a sweep is *shape*. A coherent gradient across a
+parameter is different evidence from a scattered set of positive points: it says the
+result varies with the parameter the way a real effect would, and it can point off the
+edge of the searched space, which a leaderboard never shows.
+"""
+
+import math
+from typing import Any, Dict, List, Optional, Sequence
+
+import numpy as np
+
+from quantred.analytics.metrics import expected_max_sharpe
+
+#: Objectives whose null hypothesis is zero and whose estimate is roughly symmetric —
+#: the conditions under which "the expected maximum of N draws" is a statement about
+#: noise rather than an arbitrary number.
+#:
+#: Declared here, beside the code that uses it, and deliberately short. A profit factor
+#: is null-centred on 1 and heavily skewed; a total return over a fixed window is
+#: skewed too. Computing this for them would produce a confident figure that means
+#: nothing, which is worse than the honest refusal below — a screen that cannot say
+#: what noise looks like for an objective must say *that*, not guess.
+NULL_CENTRED_OBJECTIVES = frozenset({"sharpe_ratio", "sortino_ratio", "information_ratio"})
+
+
+def score_distribution(scores: Sequence[float]) -> Dict[str, Any]:
+    """Where a sweep's results actually sit, before anything is ranked.
+
+    ``n_finite`` is reported separately from ``n``: an evaluation that failed or
+    produced no trades is not a result of zero, and silently dropping it would make a
+    sweep look narrower and more successful than it was.
+    """
+    values = np.asarray(list(scores), dtype="float64")
+    finite = values[np.isfinite(values)]
+    summary: Dict[str, Any] = {
+        "n": int(values.size),
+        "n_finite": int(finite.size),
+        "n_dropped": int(values.size - finite.size),
+    }
+    if finite.size == 0:
+        summary.update(
+            {
+                "median": None,
+                "mean": None,
+                "std": None,
+                "p25": None,
+                "p75": None,
+                "min": None,
+                "max": None,
+                "positive_rate": None,
+            }
+        )
+        return summary
+    summary.update(
+        {
+            "median": float(np.median(finite)),
+            "mean": float(np.mean(finite)),
+            # Sample standard deviation: with one point there is no spread to report,
+            # and 0.0 would read as "every result agreed".
+            "std": float(np.std(finite, ddof=1)) if finite.size > 1 else None,
+            "p25": float(np.percentile(finite, 25)),
+            "p75": float(np.percentile(finite, 75)),
+            "min": float(np.min(finite)),
+            "max": float(np.max(finite)),
+            "positive_rate": float(np.mean(finite > 0)),
+        }
+    )
+    return summary
+
+
+def noise_baseline(scores: Sequence[float], objective: str) -> Dict[str, Any]:
+    """What the best of this many draws would be worth if none of them had any edge.
+
+    The dispersion is taken from the screened results themselves, which is what the
+    Deflated Sharpe's own correction asks for: the variance of the estimates across the
+    configurations that were tried. Two things about that are worth stating plainly,
+    and both are carried in the returned payload rather than left to a reader:
+
+    * It assumes the draws are **independent**. Neighbouring grid points are not — they
+      share most of their parameters and most of their trades — so the effective number
+      of independent trials is smaller than ``n``, and this baseline is correspondingly
+      *higher* than a correlation-aware one would be.
+    * The dispersion is measured on results that may contain real structure, so it is
+      not a pure-noise dispersion either.
+
+    It is a reference point for reading a leaderboard, not a test. ``applicable`` is
+    ``False`` — with the reason attached — whenever the objective's null is not zero,
+    because a number computed anyway would be quoted as though it meant something.
+    """
+    values = np.asarray(list(scores), dtype="float64")
+    finite = values[np.isfinite(values)]
+    payload: Dict[str, Any] = {
+        "objective": objective,
+        "n_draws": int(finite.size),
+        "applicable": False,
+        "reason": "",
+        "dispersion": None,
+        "expected_best_under_null": None,
+        "observed_best": float(np.max(finite)) if finite.size else None,
+        "assumes_independence": True,
+    }
+    if objective not in NULL_CENTRED_OBJECTIVES:
+        payload["reason"] = (
+            f"no null baseline for {objective}: the maximum of N draws is only "
+            f"interpretable for an objective whose null is zero and roughly symmetric "
+            f"({', '.join(sorted(NULL_CENTRED_OBJECTIVES))}). The best point here is "
+            f"still the best of {int(finite.size)} draws — nothing here says what that "
+            f"is worth."
+        )
+        return payload
+    if finite.size < 2:
+        payload["reason"] = "fewer than two evaluated points: nothing to measure a spread from"
+        return payload
+    variance = float(np.var(finite, ddof=1))
+    if variance <= 0:
+        payload["reason"] = "every evaluated point scored identically, so there is no spread to draw from"
+        return payload
+    payload.update(
+        {
+            "applicable": True,
+            "dispersion": math.sqrt(variance),
+            "expected_best_under_null": float(expected_max_sharpe(variance, int(finite.size))),
+        }
+    )
+    return payload
+
+
+def parameter_gradient(
+    rows: Sequence[Dict[str, Any]], parameter: str, objective: str
+) -> Optional[List[Dict[str, Any]]]:
+    """How the objective behaves across one parameter's values.
+
+    The question a leaderboard cannot answer. A positive rate that falls monotonically
+    as a filter tightens is structure; the same number of positive points scattered at
+    random across the axis is not, and the two produce identical winners.
+
+    It also shows where the evidence points *off the edge* of what was searched, which
+    is the finding a best-point report structurally cannot produce.
+
+    ``None`` when the parameter did not vary — a constant column has no gradient, and
+    reporting a single row would invite reading one value as a trend.
+    """
+    buckets: Dict[Any, List[float]] = {}
+    for row in rows:
+        if parameter not in row:
+            continue
+        score = row.get(objective)
+        if score is None or not math.isfinite(float(score)):
+            continue
+        buckets.setdefault(row[parameter], []).append(float(score))
+    if len(buckets) < 2:
+        return None
+    out = []
+    for value in sorted(buckets, key=lambda v: (isinstance(v, str), v)):
+        scores = buckets[value]
+        out.append(
+            {
+                "value": value,
+                "n": len(scores),
+                "positive": int(sum(1 for s in scores if s > 0)),
+                "positive_rate": float(sum(1 for s in scores if s > 0) / len(scores)),
+                "median": float(np.median(scores)),
+                "max": float(np.max(scores)),
+            }
+        )
+    return out
+
+
+def gradients(
+    rows: Sequence[Dict[str, Any]], parameters: Sequence[str], objective: str
+) -> Dict[str, List[Dict[str, Any]]]:
+    """:func:`parameter_gradient` for every parameter that actually varied."""
+    out = {}
+    for parameter in parameters:
+        gradient = parameter_gradient(rows, parameter, objective)
+        if gradient is not None:
+            out[parameter] = gradient
+    return out

quantred/analytics/series_comparison.py

@@ -0,0 +1,303 @@
+"""Are two recorded results the same result?
+
+A campaign accumulates trials that look different — different parameters, different
+kinds, different windows — and some of them are the same trade expressed twice. Two
+candidates correlating at 0.97 are one candidate, and promoting both as independent
+evidence double-counts a single bet. Answering that meant pulling both return series
+out of SQLite by hand and correlating them in a notebook.
+
+Everything here is a pure function over stored series and returns structured data. It
+grades nothing. What it is opinionated about is **refusing**, because a correlation is
+a claim about a relationship and there is no honest partial version of one:
+
+* **Below the minimum overlap, a pair is refused.** Not computed-and-caveated. A
+  correlation over a handful of shared dates is a number with an error bar wider than
+  its own range, and printed to two decimals it looks exactly like a measurement.
+* **Across accounting versions, a pair is refused by default.** The two series were
+  produced by engines that compute different things; their correlation is a fact about
+  the instruments as much as the strategies. Forcing it is allowed and the pair is
+  labelled incomparable when you do.
+* **Every correlation carries its interval.** Reported with a Fisher-z confidence
+  interval, so a coefficient resting on a short overlap arrives visibly wide rather
+  than merely short of decimals.
+
+The pairwise matrix alone would hide all of that — a refusal and a correlation of zero
+occupy the same cell — so the per-pair diagnostics travel beside it and the matrix
+holds ``None`` where nothing was computed.
+"""
+
+import math
+from typing import Any, Dict, List, Optional, Sequence
+
+import numpy as np
+
+from quantred.analytics import metrics
+
+#: Shared dates below which a pair is refused rather than reported. Matches the trial
+#: store's own panel default: the number that makes a joint resampling meaningful is
+#: the same number that makes a pairwise correlation meaningful.
+MIN_OVERLAP = 60
+
+#: Confidence level for the interval attached to every correlation.
+CONFIDENCE = 0.95
+
+COMPARED = "compared"
+REFUSED = "refused"
+
+
+def series_summary(entry: Dict[str, Any]) -> Dict[str, Any]:
+    """What one trial's stored series is, before anything is compared to it."""
+    trial_id = entry.get("trial_id")
+    dates, values = entry.get("dates"), entry.get("values")
+    summary: Dict[str, Any] = {
+        "trial_id": trial_id,
+        "accounting": entry.get("accounting"),
+        "strategy": entry.get("strategy"),
+        "kind": entry.get("kind"),
+        "available": False,
+        "reason": "",
+        "periods": None,
+        "start": None,
+        "end": None,
+    }
+    if dates is None or values is None:
+        summary["reason"] = (
+            "no return series was recorded for this trial — not every trial kind persists "
+            "one, and trials predating the companion table have none"
+        )
+        return summary
+    if not dates:
+        # Distinct from the line above on purpose. "Nothing was stored" and "a series
+        # was stored and it is empty" are different facts about the record.
+        summary["reason"] = "the stored return series is empty"
+        summary["periods"] = 0
+        return summary
+
+    arr = np.asarray(values, dtype="float64")
+    finite = arr[np.isfinite(arr)]
+    summary.update(
+        {
+            "available": True,
+            "periods": len(dates),
+            "start": str(dates[0]),
+            "end": str(dates[-1]),
+            "n_finite": int(finite.size),
+            "n_unusable": int(arr.size - finite.size),
+            "annualized_sharpe": float(metrics.sharpe_ratio(finite)) if finite.size > 1 else None,
+            "annualized_volatility": (
+                float(metrics.annualized_volatility(finite)) if finite.size > 1 else None
+            ),
+            "cumulative_return": float(np.prod(1.0 + finite) - 1.0) if finite.size else None,
+        }
+    )
+    return summary
+
+
+def _correlation_interval(r: float, n: int) -> Optional[Dict[str, float]]:
+    """A Fisher-z interval around a correlation coefficient.
+
+    The point of carrying it is that it does not shrink politely: a coefficient
+    resting on a short overlap arrives visibly wide, which is the honest rendering of
+    "we do not really know". ``None`` under four observations, where the transform has
+    no standard error to work with at all.
+    """
+    if n < 4 or not -1.0 < r < 1.0:
+        # No interval exists at |r| = 1: the transform is undefined there, and it is
+        # undefined because the answer is not uncertain. Callers say which of the two
+        # reasons applies rather than reporting a missing interval as thin evidence.
+        return None
+    z = math.atanh(r)
+    se = 1.0 / math.sqrt(n - 3)
+    half = metrics.norm_ppf(1.0 - (1.0 - CONFIDENCE) / 2.0) * se
+    return {"low": float(math.tanh(z - half)), "high": float(math.tanh(z + half))}
+
+
+def _pair(
+    left: Dict[str, Any],
+    right: Dict[str, Any],
+    *,
+    min_overlap: int,
+    across_accounting: bool,
+) -> Dict[str, Any]:
+    """One pair: the overlap they share, and the correlation over it or the refusal."""
+    out: Dict[str, Any] = {
+        "a": left.get("trial_id"),
+        "b": right.get("trial_id"),
+        "status": REFUSED,
+        "reason": "",
+        "overlap": 0,
+        "start": None,
+        "end": None,
+        "correlation": None,
+        "interval": None,
+    }
+
+    for side in (left, right):
+        if not (side.get("dates") and side.get("values")):
+            out["reason"] = f"{side.get('trial_id')} has no recorded return series to compare"
+            return out
+
+    acc_left, acc_right = left.get("accounting"), right.get("accounting")
+    if acc_left != acc_right and not across_accounting:
+        out["reason"] = (
+            f"recorded under accounting v{acc_left} and v{acc_right}. The two series were "
+            "produced by engines that compute different things, so their correlation is "
+            "partly a fact about the instruments. Pass the cross-accounting option to "
+            "compare them anyway"
+        )
+        return out
+
+    a = dict(zip((str(d) for d in left["dates"]), left["values"]))
+    b = dict(zip((str(d) for d in right["dates"]), right["values"]))
+    shared = sorted(set(a) & set(b))
+    x = np.asarray([a[d] for d in shared], dtype="float64")
+    y = np.asarray([b[d] for d in shared], dtype="float64")
+    usable = np.isfinite(x) & np.isfinite(y)
+    x, y = x[usable], y[usable]
+    dates = [d for d, keep in zip(shared, usable) if keep]
+
+    out["overlap"] = int(x.size)
+    out["start"] = dates[0] if dates else None
+    out["end"] = dates[-1] if dates else None
+
+    if x.size < min_overlap:
+        out["reason"] = (
+            f"{x.size} shared dates, below the {min_overlap} this comparison requires. A "
+            "correlation over that little is an error bar, not a measurement"
+        )
+        return out
+    if float(np.std(x)) == 0.0 or float(np.std(y)) == 0.0:
+        out["reason"] = "one series does not vary over the shared dates, so nothing can correlate"
+        return out
+
+    r = float(np.corrcoef(x, y)[0, 1])
+    out.update(
+        {
+            "status": COMPARED,
+            "correlation": r,
+            "interval": _correlation_interval(r, int(x.size)),
+            # Why there is no interval, when there is none. "Undefined at |r| = 1" and
+            # "too few observations to estimate one" are opposite statements about how
+            # much is known, and a bare missing interval reads as the second.
+            "interval_note": (
+                "no interval: the two series are perfectly correlated over the shared "
+                "dates, which is a stronger statement than any interval, not a weaker one"
+                if abs(r) >= 1.0
+                else "no interval: fewer than four shared observations"
+                if x.size < 4
+                else ""
+            ),
+            "comparable": acc_left == acc_right,
+        }
+    )
+    if acc_left != acc_right:
+        out["reason"] = (
+            f"INCOMPARABLE as evidence: accounting v{acc_left} against v{acc_right}. "
+            "Computed because it was asked for"
+        )
+    return out
+
+
+def _deduplicate(entries: Sequence[Dict[str, Any]]) -> "tuple[List[Dict[str, Any]], List[str]]":
+    """Entries with repeated trial ids collapsed to their first occurrence.
+
+    A trial correlated against itself is 1.0 by construction and says nothing, and the
+    matrix below is keyed by trial id - so a repeat silently overwrote one cell and
+    left another unwritten. Reported rather than dropped quietly: asking about three
+    ids and being answered about two is worth a line.
+    """
+    seen: Dict[str, Dict[str, Any]] = {}
+    duplicates: List[str] = []
+    for entry in entries:
+        trial_id = entry.get("trial_id")
+        if trial_id in seen:
+            duplicates.append(trial_id)
+            continue
+        seen[trial_id] = entry
+    return list(seen.values()), duplicates
+
+
+#: What a stored return series *is*, by trial kind. A walk-forward's is the out-of-sample
+#: folds stitched together; a backtest's is one pass over its whole window, which is
+#: in-sample. Correlating the two is arithmetic over unlike things, so every comparison
+#: names each series' kind and says when the kinds differ.
+SERIES_BASIS = {
+    "walkforward": "stitched out-of-sample folds",
+    "research": "stitched out-of-sample folds",
+    "backtest": "full-window, in-sample",
+    "optimize": "full-window, in-sample",
+}
+
+
+def series_basis(kind: Optional[str]) -> Optional[str]:
+    """What a series of this trial kind measures, or ``None`` for a kind not listed."""
+    return SERIES_BASIS.get(kind or "")
+
+
+def compare_series(
+    entries: Sequence[Dict[str, Any]],
+    *,
+    min_overlap: int = MIN_OVERLAP,
+    across_accounting: bool = False,
+) -> Dict[str, Any]:
+    """Every pair among the given trials' recorded return series.
+
+    ``entries`` are ``{trial_id, dates, values, accounting, strategy, kind}``. The
+    result carries the per-series summaries, one diagnostic per pair, and the matrix —
+    in that order deliberately, because the matrix is the part that reads as an answer
+    and the refusals are the part that says how much of one it is.
+    """
+    # Deduplicated first, and the duplicates counted. The same id given twice used to
+    # collapse in the matrix index below, so a pair that *was* compared got written to
+    # one cell and the other was left holding None - which this module defines as a
+    # refusal. A paste slip was enough to make the matrix say two results are
+    # incomparable when they had just been compared.
+    entries, duplicates = _deduplicate(entries)
+    summaries = [series_summary(entry) for entry in entries]
+    ids = [entry.get("trial_id") for entry in entries]
+
+    pairs: List[Dict[str, Any]] = []
+    for i in range(len(entries)):
+        for j in range(i + 1, len(entries)):
+            pairs.append(
+                _pair(
+                    entries[i],
+                    entries[j],
+                    min_overlap=min_overlap,
+                    across_accounting=across_accounting,
+                )
+            )
+
+    # ``None`` where nothing was computed, never 0.0. In a correlation matrix a zero
+    # is a strong claim - these two results move independently - and it is the exact
+    # claim a refusal is unable to make.
+    index = {tid: n for n, tid in enumerate(ids)}
+    matrix: List[List[Optional[float]]] = [
+        [1.0 if i == j else None for j in range(len(ids))] for i in range(len(ids))
+    ]
+    for pair in pairs:
+        if pair["status"] != COMPARED:
+            continue
+        i, j = index[pair["a"]], index[pair["b"]]
+        matrix[i][j] = matrix[j][i] = pair["correlation"]
+
+    compared = [p for p in pairs if p["status"] == COMPARED]
+    kind_of = {entry.get("trial_id"): entry.get("kind") for entry in entries}
+    for pair in pairs:
+        pair["kind_a"], pair["kind_b"] = kind_of.get(pair["a"]), kind_of.get(pair["b"])
+        # Unlike bases, not unlike kinds: two walk-forwards and a research round are all
+        # out-of-sample series, and a note on every such pair would be noise.
+        pair["unlike_basis"] = series_basis(pair["kind_a"]) != series_basis(pair["kind_b"])
+    return {
+        "kinds_differ": len({series_basis(k) for k in kind_of.values()}) > 1,
+        "duplicate_trial_ids": duplicates,
+        "min_overlap": min_overlap,
+        "across_accounting": across_accounting,
+        "series": summaries,
+        "pairs": pairs,
+        "matrix": {"trial_ids": ids, "values": matrix},
+        "n_pairs": len(pairs),
+        "n_compared": len(compared),
+        "n_refused": len(pairs) - len(compared),
+        "highest": (max(compared, key=lambda p: p["correlation"]) if compared else None),
+    }