finval 0.1.0__py3-none-any.whl

finval/__init__.py

@@ -0,0 +1,29 @@
+"""finval — rigorous validation for synthetic financial time series.
+
+Quick start:
+
+    import finval
+
+    # synthetic and real are (n_samples, n_features) arrays of returns
+    report = finval.validate(synthetic, real)
+    print(report.summary())
+    report.to_dict()
+
+For path-level validation with drawdowns and calibration:
+
+    # paths are (n_paths, horizon, n_features) arrays
+    report = finval.validate_paths(synthetic_paths, real_returns)
+"""
+
+from finval.core.result import MetricResult, ValidationReport
+from finval.validate import validate, validate_paths
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MetricResult",
+    "ValidationReport",
+    "validate",
+    "validate_paths",
+    "__version__",
+]

finval/baselines/__init__.py

@@ -0,0 +1,32 @@
+"""Baseline generators for benchmarking synthetic financial data.
+
+A baseline is a simple generator that produces synthetic data from real
+data. Baselines are not meant to be good models — they're meant to be
+reference points so users can answer "is my fancy model actually better
+than X?" where X is a trivially simple generator.
+
+Three baselines ship with finval:
+
+- `gaussian`: Multivariate Gaussian with the empirical mean and covariance
+  of the real data. No temporal structure, no tails, no vol clustering.
+  This is the minimum bar any generative model should clear.
+
+- `historical_bootstrap`: Random sampling (with replacement) from real
+  returns. Reproduces marginals and joint distribution exactly in
+  expectation, but destroys all temporal structure (no ACF, no leverage).
+
+- `block_bootstrap`: Moving-block bootstrap preserves local dependence.
+  This is the strongest simple baseline — it reproduces short-range
+  temporal structure and most stylized facts at the cost of exact
+  marginal duplication. A generative model that beats block bootstrap
+  on all metrics is genuinely doing something new.
+"""
+
+from finval.baselines.gaussian import gaussian_baseline
+from finval.baselines.historical import block_bootstrap, historical_bootstrap
+
+__all__ = [
+    "gaussian_baseline",
+    "historical_bootstrap",
+    "block_bootstrap",
+]

finval/baselines/gaussian.py

@@ -0,0 +1,64 @@
+"""Multivariate Gaussian baseline.
+
+Fits a multivariate normal to the real returns and samples from it.
+This is the naive "Gaussian i.i.d." benchmark. A model that can't beat
+this on temporal metrics is not capturing any time structure.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def gaussian_baseline(
+    real: np.ndarray,
+    n_samples: int | None = None,
+    n_paths: int | None = None,
+    path_length: int | None = None,
+    seed: int = 42,
+) -> np.ndarray:
+    """Generate samples from a multivariate Gaussian fit to real returns.
+
+    Two output modes:
+
+    1. Flat mode (n_samples given): returns shape (n_samples, n_features),
+       matching the format expected by distribution / dependence metrics.
+
+    2. Path mode (n_paths and path_length given): returns shape
+       (n_paths, path_length, n_features), matching the format expected
+       by temporal and path metrics. Each row within a path is an
+       independent draw — there is no temporal structure.
+
+    Args:
+        real: (n_obs, n_features) real return series used to fit mean+cov.
+        n_samples: Number of flat samples to return.
+        n_paths: Number of paths to return (requires path_length).
+        path_length: Length of each path.
+        seed: RNG seed.
+
+    Returns:
+        numpy array of shape (n_samples, n_features) or (n_paths, path_length, n_features).
+    """
+    real = np.asarray(real)
+    if real.ndim != 2:
+        raise ValueError(f"real must be 2D, got shape {real.shape}")
+
+    mean = np.nanmean(real, axis=0)
+    # Handle NaN by dropping rows before covariance
+    clean = real[~np.any(np.isnan(real), axis=1)]
+    if len(clean) < 2:
+        raise ValueError("need at least 2 clean rows to fit covariance")
+    cov = np.cov(clean, rowvar=False)
+    # Regularize to ensure positive-definite
+    d = cov.shape[0]
+    cov = cov + 1e-10 * np.eye(d)
+
+    rng = np.random.default_rng(seed)
+
+    if n_samples is not None:
+        return rng.multivariate_normal(mean, cov, size=n_samples)
+
+    if n_paths is not None and path_length is not None:
+        return rng.multivariate_normal(mean, cov, size=(n_paths, path_length))
+
+    raise ValueError("must specify either n_samples or (n_paths and path_length)")

finval/baselines/historical.py

@@ -0,0 +1,80 @@
+"""Historical resampling baselines.
+
+- `historical_bootstrap` samples rows i.i.d. from real returns. Preserves
+  the empirical marginal and joint distribution perfectly but destroys
+  all temporal structure.
+
+- `block_bootstrap` samples contiguous blocks of rows. Preserves short-
+  range temporal dependence (within a block) at the cost of some joint
+  distribution fidelity at block boundaries. This is the strongest simple
+  baseline for temporal stylized facts.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def historical_bootstrap(
+    real: np.ndarray,
+    n_samples: int | None = None,
+    n_paths: int | None = None,
+    path_length: int | None = None,
+    seed: int = 42,
+) -> np.ndarray:
+    """I.i.d. bootstrap: sample rows of real with replacement.
+
+    Perfect marginal and joint distribution match in expectation.
+    Zero temporal structure (independent rows).
+    """
+    real = np.asarray(real)
+    if real.ndim != 2:
+        raise ValueError(f"real must be 2D, got shape {real.shape}")
+
+    rng = np.random.default_rng(seed)
+    n = len(real)
+
+    if n_samples is not None:
+        idx = rng.integers(0, n, size=n_samples)
+        return real[idx]
+
+    if n_paths is not None and path_length is not None:
+        idx = rng.integers(0, n, size=(n_paths, path_length))
+        return real[idx]
+
+    raise ValueError("must specify either n_samples or (n_paths and path_length)")
+
+
+def block_bootstrap(
+    real: np.ndarray,
+    n_paths: int,
+    path_length: int,
+    block_size: int = 20,
+    seed: int = 42,
+) -> np.ndarray:
+    """Moving-block bootstrap: sample contiguous blocks of rows.
+
+    Preserves temporal dependence up to block_size. A block size of
+    ~20 trading days is typical for financial data. Only supports path
+    output (the concept doesn't make sense for flat samples).
+
+    Returns shape (n_paths, path_length, n_features).
+    """
+    real = np.asarray(real)
+    if real.ndim != 2:
+        raise ValueError(f"real must be 2D, got shape {real.shape}")
+    if len(real) < block_size:
+        raise ValueError(f"real has {len(real)} rows, need >= block_size={block_size}")
+
+    rng = np.random.default_rng(seed)
+    n = len(real)
+    n_features = real.shape[1]
+    n_blocks = (path_length + block_size - 1) // block_size
+
+    paths = np.empty((n_paths, path_length, n_features), dtype=real.dtype)
+    for p in range(n_paths):
+        starts = rng.integers(0, n - block_size + 1, size=n_blocks)
+        blocks = [real[s : s + block_size] for s in starts]
+        full = np.concatenate(blocks, axis=0)
+        paths[p] = full[:path_length]
+    return paths

finval/core/__init__.py

@@ -0,0 +1,11 @@
+"""Core types and utilities for finval."""
+
+from finval.core.result import MetricResult, ValidationReport
+from finval.core.thresholds import DEFAULT_THRESHOLDS, quality_from_value
+
+__all__ = [
+    "MetricResult",
+    "ValidationReport",
+    "DEFAULT_THRESHOLDS",
+    "quality_from_value",
+]

finval/core/bootstrap.py

@@ -0,0 +1,116 @@
+"""Bootstrap confidence intervals for metrics.
+
+Block bootstrap is appropriate for time series (preserves local dependence).
+Naive i.i.d. bootstrap is appropriate for flattened-sample metrics like
+marginal KS or Pearson correlation.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import numpy as np
+
+
+def iid_bootstrap_ci(
+    metric_fn: Callable[[np.ndarray, np.ndarray], float],
+    synthetic: np.ndarray,
+    real: np.ndarray,
+    n_bootstrap: int = 200,
+    confidence: float = 0.95,
+    seed: int = 42,
+) -> tuple[float, float]:
+    """Compute a bootstrap confidence interval for a metric under i.i.d. resampling.
+
+    This is valid for cross-sectional metrics where rows are treated as
+    exchangeable samples (e.g., marginal KS, pairwise correlation error).
+
+    Args:
+        metric_fn: Callable taking (synthetic, real) arrays and returning
+            a scalar metric value.
+        synthetic: (n_samples, n_features) synthetic data.
+        real: (n_samples, n_features) real data.
+        n_bootstrap: Number of bootstrap replicates.
+        confidence: Desired confidence level, e.g. 0.95 for a 95% CI.
+        seed: RNG seed for reproducibility.
+
+    Returns:
+        (ci_low, ci_high) for the metric.
+    """
+    rng = np.random.default_rng(seed)
+    n_syn = len(synthetic)
+    n_real = len(real)
+
+    values: list[float] = []
+    for _ in range(n_bootstrap):
+        idx_syn = rng.integers(0, n_syn, size=n_syn)
+        idx_real = rng.integers(0, n_real, size=n_real)
+        try:
+            v = metric_fn(synthetic[idx_syn], real[idx_real])
+            if np.isfinite(v):
+                values.append(float(v))
+        except Exception:
+            continue
+
+    if not values:
+        return (float("nan"), float("nan"))
+
+    alpha = (1 - confidence) / 2
+    lo = float(np.percentile(values, alpha * 100))
+    hi = float(np.percentile(values, (1 - alpha) * 100))
+    return (lo, hi)
+
+
+def block_bootstrap_ci(
+    metric_fn: Callable[[np.ndarray, np.ndarray], float],
+    synthetic: np.ndarray,
+    real: np.ndarray,
+    block_size: int = 20,
+    n_bootstrap: int = 200,
+    confidence: float = 0.95,
+    seed: int = 42,
+) -> tuple[float, float]:
+    """Compute a bootstrap CI using moving-block bootstrap for time series.
+
+    Preserves local temporal dependence by resampling contiguous blocks.
+    Appropriate for metrics that depend on the order of samples (ACF,
+    volatility clustering, leverage effect).
+
+    Args:
+        metric_fn: Callable taking (synthetic, real) arrays and returning
+            a scalar metric value.
+        synthetic: (n_timesteps, n_features) synthetic time series.
+        real: (n_timesteps, n_features) real time series.
+        block_size: Length of each resampling block.
+        n_bootstrap: Number of bootstrap replicates.
+        confidence: Desired confidence level.
+        seed: RNG seed.
+
+    Returns:
+        (ci_low, ci_high).
+    """
+    rng = np.random.default_rng(seed)
+
+    def resample(x: np.ndarray) -> np.ndarray:
+        n = len(x)
+        n_blocks = (n + block_size - 1) // block_size
+        starts = rng.integers(0, max(n - block_size + 1, 1), size=n_blocks)
+        out = np.concatenate([x[s : s + block_size] for s in starts], axis=0)
+        return out[:n]
+
+    values: list[float] = []
+    for _ in range(n_bootstrap):
+        try:
+            v = metric_fn(resample(synthetic), resample(real))
+            if np.isfinite(v):
+                values.append(float(v))
+        except Exception:
+            continue
+
+    if not values:
+        return (float("nan"), float("nan"))
+
+    alpha = (1 - confidence) / 2
+    lo = float(np.percentile(values, alpha * 100))
+    hi = float(np.percentile(values, (1 - alpha) * 100))
+    return (lo, hi)

finval/core/result.py

@@ -0,0 +1,201 @@
+"""Result types for validation metrics.
+
+A `MetricResult` captures the output of a single metric: value, quality
+grade, thresholds used, and optional bootstrap confidence interval and
+per-feature/per-pair breakdown.
+
+A `ValidationReport` aggregates multiple MetricResults into a single object
+with an overall quality grade and weighted pass rate.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+QUALITY_LEVELS = ("excellent", "good", "acceptable", "poor")
+QUALITY_SCORES = {"excellent": 1.0, "good": 0.75, "acceptable": 0.5, "poor": 0.0}
+
+
+@dataclass
+class MetricResult:
+    """Result of a single validation metric.
+
+    Attributes:
+        name: Metric identifier, e.g. "marginal_ks".
+        value: Scalar metric value (lower is always better by convention).
+        quality: One of "excellent", "good", "acceptable", "poor".
+        passed: True if quality is not "poor".
+        thresholds: The threshold dict used to assign quality.
+        category: Metric category, e.g. "distribution", "dependence".
+        interpretation: Human-readable one-line summary.
+        ci_low: Lower bound of bootstrap confidence interval (optional).
+        ci_high: Upper bound of bootstrap confidence interval (optional).
+        per_feature: Per-feature breakdown (optional).
+        per_pair: Per-pair breakdown for dependence metrics (optional).
+        metadata: Additional metric-specific information.
+    """
+
+    name: str
+    value: float
+    quality: str
+    passed: bool
+    thresholds: dict[str, float] = field(default_factory=dict)
+    category: str = "uncategorized"
+    interpretation: str = ""
+    ci_low: float | None = None
+    ci_high: float | None = None
+    per_feature: dict[str, Any] = field(default_factory=dict)
+    per_pair: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        # Sanitize NaN/Inf to ensure serializable output
+        if self.value is not None and (np.isnan(self.value) or np.isinf(self.value)):
+            self.value = float("inf")
+            self.quality = "poor"
+            self.passed = False
+        if self.quality not in QUALITY_LEVELS:
+            raise ValueError(f"quality must be one of {QUALITY_LEVELS}, got {self.quality!r}")
+
+    @property
+    def score(self) -> float:
+        """Numeric score in [0, 1] for weighted aggregation."""
+        return QUALITY_SCORES[self.quality]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict (JSON-safe)."""
+        out: dict[str, Any] = {
+            "name": self.name,
+            "value": self.value,
+            "quality": self.quality,
+            "passed": self.passed,
+            "score": self.score,
+            "thresholds": self.thresholds,
+            "category": self.category,
+            "interpretation": self.interpretation,
+        }
+        if self.ci_low is not None:
+            out["ci_low"] = self.ci_low
+            out["ci_high"] = self.ci_high
+        if self.per_feature:
+            out["per_feature"] = self.per_feature
+        if self.per_pair:
+            out["per_pair"] = self.per_pair
+        if self.metadata:
+            out["metadata"] = self.metadata
+        return out
+
+
+def create_error_metric(
+    name: str,
+    error: str,
+    category: str = "uncategorized",
+) -> MetricResult:
+    """Build a poor-quality result representing a computation failure."""
+    return MetricResult(
+        name=name,
+        value=float("inf"),
+        quality="poor",
+        passed=False,
+        category=category,
+        interpretation=f"Failed: {error}",
+        metadata={"error": error},
+    )
+
+
+@dataclass
+class ValidationReport:
+    """Aggregated report of multiple validation metrics.
+
+    A ValidationReport groups metrics by category, computes a weighted
+    overall score, and assigns an overall quality grade.
+
+    Attributes:
+        metrics: Dict mapping metric name to MetricResult.
+        weights: Dict mapping metric name to absolute weight (fractions
+            should sum to ~1 over included metrics; missing metrics
+            contribute 0).
+        category_weights: Dict mapping category to category weight.
+        overall_score: Weighted sum of metric scores in [0, 1].
+        overall_quality: One of "excellent" (>=0.85), "good" (>=0.65),
+            "acceptable" (>=0.45), "poor" otherwise.
+    """
+
+    metrics: dict[str, MetricResult] = field(default_factory=dict)
+    weights: dict[str, float] = field(default_factory=dict)
+    category_weights: dict[str, float] = field(default_factory=dict)
+
+    @property
+    def overall_score(self) -> float:
+        """Weighted average of metric scores. Weights for missing metrics
+        contribute zero, which penalizes incomplete validation runs."""
+        if not self.metrics or not self.weights:
+            return 0.0
+        total_weight = sum(self.weights.values())
+        if total_weight == 0:
+            return 0.0
+        weighted = sum(
+            m.score * self.weights.get(m.name, 0.0) for m in self.metrics.values()
+        )
+        return weighted / total_weight
+
+    @property
+    def overall_quality(self) -> str:
+        """Quality grade based on overall_score."""
+        s = self.overall_score
+        if s >= 0.85:
+            return "excellent"
+        if s >= 0.65:
+            return "good"
+        if s >= 0.45:
+            return "acceptable"
+        return "poor"
+
+    @property
+    def pass_rate(self) -> float:
+        """Fraction of metrics with quality >= acceptable."""
+        if not self.metrics:
+            return 0.0
+        passed = sum(1 for m in self.metrics.values() if m.passed)
+        return passed / len(self.metrics)
+
+    def by_category(self) -> dict[str, list[MetricResult]]:
+        """Group metrics by category."""
+        out: dict[str, list[MetricResult]] = {}
+        for m in self.metrics.values():
+            out.setdefault(m.category, []).append(m)
+        return out
+
+    def summary(self) -> str:
+        """Human-readable one-page summary."""
+        lines = [
+            f"finval ValidationReport — {self.overall_quality.upper()} ({self.overall_score:.0%})",
+            f"  metrics: {len(self.metrics)}, passed: {int(self.pass_rate * len(self.metrics))}/{len(self.metrics)}",
+            "",
+        ]
+        for category, mlist in self.by_category().items():
+            lines.append(f"  [{category}]")
+            for m in sorted(mlist, key=lambda x: x.name):
+                status = "PASS" if m.passed else "FAIL"
+                ci = ""
+                if m.ci_low is not None:
+                    ci = f" ({m.ci_low:.3f}–{m.ci_high:.3f})"
+                lines.append(
+                    f"    {status}  {m.name:28s}  value={m.value:7.4f}{ci}  {m.quality}"
+                )
+            lines.append("")
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict (JSON-safe)."""
+        return {
+            "overall_score": self.overall_score,
+            "overall_quality": self.overall_quality,
+            "pass_rate": self.pass_rate,
+            "metrics": {name: m.to_dict() for name, m in self.metrics.items()},
+            "weights": dict(self.weights),
+            "category_weights": dict(self.category_weights),
+        }