alpha-engine-lib 0.32.0__py3-none-any.whl

alpha_engine_lib/reconcile.py

@@ -0,0 +1,203 @@
+"""
+Quantitative parity reconciliation for ``dict[str, DataFrame]`` price stores.
+
+The SOTA observation substrate for contract-safe data-tier migrations: when a
+consumer is being moved from one price source to another (e.g. the
+``predictor/price_cache_slim/`` parquet tier -> the ArcticDB universe lib,
+Wave 4 of the predictor/ S3 namespace rationalization), the cutover decision
+must be **data-driven**, not eyeballed. This module turns "do the two sources
+agree?" into a single auditable :class:`ParityReport` with:
+
+- ticker-set symmetric difference (coverage),
+- per-ticker row-count delta (shape),
+- max absolute value delta over the *overlapping* dates/columns (fidelity),
+- a binary ``passed`` keyed on an explicit epsilon.
+
+:meth:`ParityReport.as_metrics` is JSON-able so the same object that decides a
+gate can be emitted to the metrics surface and observed over a window before
+the producer is retired. One implementation, reused by every consumer
+migration PR (data macro-breadth, backtester exit-timing) and the final
+deletion gate — no per-repo re-implementation of "are these prices equal".
+
+Pure pandas; importable without the ``[arcticdb]`` extra (pandas is imported
+lazily inside the function, mirroring the arcticdb module's contract).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping, Optional, Sequence, Tuple
+
+
+@dataclass(frozen=True)
+class ParityReport:
+    """Outcome of comparing two ticker -> DataFrame price stores.
+
+    ``passed`` is the gate signal. ``as_metrics()`` is what you log over the
+    observation window. ``summary()`` is the one-line operator string.
+    """
+
+    only_in_a: frozenset
+    only_in_b: frozenset
+    common: frozenset
+    rowcount_deltas: Mapping[str, int]  # ticker -> len(a)-len(b), nonzero only
+    max_abs_value_delta: float
+    worst_cell: Optional[Tuple[str, str, str]]  # (ticker, column, iso-date)
+    n_cells_over_epsilon: int
+    n_cells_compared: int
+    epsilon: float
+    value_cols: Tuple[str, ...]
+    require_ticker_match: bool
+    require_rowcount_match: bool
+
+    @property
+    def ticker_sets_match(self) -> bool:
+        return not self.only_in_a and not self.only_in_b
+
+    @property
+    def rowcounts_match(self) -> bool:
+        return not self.rowcount_deltas
+
+    @property
+    def passed(self) -> bool:
+        ok = self.n_cells_over_epsilon == 0
+        if self.require_ticker_match:
+            ok = ok and self.ticker_sets_match
+        if self.require_rowcount_match:
+            ok = ok and self.rowcounts_match
+        return ok
+
+    def summary(self) -> str:
+        return (
+            "parity {verdict}: common={c} only_a={oa} only_b={ob} "
+            "rowcount_mismatch_tickers={rc} max_abs_delta={mad:.3e} "
+            "cells_over_eps={oe}/{tot} (eps={eps:.1e})"
+        ).format(
+            verdict="PASS" if self.passed else "FAIL",
+            c=len(self.common),
+            oa=len(self.only_in_a),
+            ob=len(self.only_in_b),
+            rc=len(self.rowcount_deltas),
+            mad=self.max_abs_value_delta,
+            oe=self.n_cells_over_epsilon,
+            tot=self.n_cells_compared,
+            eps=self.epsilon,
+        )
+
+    def as_metrics(self) -> dict:
+        """JSON-able metric dict for the observation gate / metrics surface."""
+        return {
+            "passed": self.passed,
+            "ticker_sets_match": self.ticker_sets_match,
+            "rowcounts_match": self.rowcounts_match,
+            "n_common": len(self.common),
+            "n_only_in_a": len(self.only_in_a),
+            "n_only_in_b": len(self.only_in_b),
+            "only_in_a": sorted(self.only_in_a),
+            "only_in_b": sorted(self.only_in_b),
+            "n_rowcount_mismatch_tickers": len(self.rowcount_deltas),
+            "rowcount_deltas": dict(sorted(self.rowcount_deltas.items())),
+            "max_abs_value_delta": self.max_abs_value_delta,
+            "worst_cell": list(self.worst_cell) if self.worst_cell else None,
+            "n_cells_over_epsilon": self.n_cells_over_epsilon,
+            "n_cells_compared": self.n_cells_compared,
+            "epsilon": self.epsilon,
+            "value_cols": list(self.value_cols),
+        }
+
+
+def reconcile_frame_dicts(
+    a: Mapping[str, "object"],
+    b: Mapping[str, "object"],
+    *,
+    value_cols: Sequence[str] = ("Close",),
+    epsilon: float = 1e-6,
+    require_ticker_match: bool = True,
+    require_rowcount_match: bool = False,
+) -> ParityReport:
+    """Compare two ticker -> DataFrame price stores into a :class:`ParityReport`.
+
+    Value fidelity is measured on the **intersection of dates** per common
+    ticker (an inner join on the DatetimeIndex), over the ``value_cols``
+    present in *both* frames. Row-count deltas are reported separately and,
+    by default, do **not** fail the gate: a slim-cache tail slice and an
+    ArcticDB ``date_range`` read legitimately differ by a few boundary rows
+    while being bit-identical on the overlap — the migration question is
+    "do they agree where they overlap", not "are the artifacts shaped
+    identically". Set ``require_rowcount_match=True`` for stricter contexts.
+
+    Args:
+        a, b: ticker -> pandas DataFrame (DatetimeIndex). Conventionally
+            ``a`` = incumbent source, ``b`` = candidate source.
+        value_cols: columns compared for numeric equality.
+        epsilon: absolute tolerance; a cell counts as a mismatch when
+            ``abs(a - b) > epsilon``.
+        require_ticker_match: include ticker-set symmetry in ``passed``.
+        require_rowcount_match: include row-count equality in ``passed``.
+    """
+    import pandas as pd  # lazy: keeps module importable without the extra
+
+    keys_a = set(a)
+    keys_b = set(b)
+    common = keys_a & keys_b
+
+    rowcount_deltas: dict = {}
+    max_abs = 0.0
+    worst_cell: Optional[Tuple[str, str, str]] = None
+    n_over = 0
+    n_compared = 0
+    cols = tuple(value_cols)
+
+    for ticker in sorted(common):
+        fa = a[ticker]
+        fb = b[ticker]
+
+        delta = len(fa) - len(fb)
+        if delta != 0:
+            rowcount_deltas[ticker] = delta
+
+        idx = fa.index.intersection(fb.index)
+        if len(idx) == 0:
+            continue
+        for col in cols:
+            if col not in fa.columns or col not in fb.columns:
+                continue
+            sa = pd.to_numeric(fa.loc[idx, col], errors="coerce")
+            sb = pd.to_numeric(fb.loc[idx, col], errors="coerce")
+            diff = (sa - sb).abs()
+            # NaN on either side -> treat as comparable only where both
+            # present; an asymmetric NaN is a real mismatch.
+            both_nan = sa.isna() & sb.isna()
+            one_nan = (sa.isna() ^ sb.isna())
+            diff = diff.where(~both_nan)
+            n_compared += int((~both_nan).sum())
+
+            over = (diff > epsilon) | one_nan
+            n_over += int(over.sum())
+
+            valid = diff.dropna()
+            if not valid.empty:
+                cell_max = float(valid.max())
+                if cell_max > max_abs:
+                    max_abs = cell_max
+                    worst_dt = valid.idxmax()
+                    worst_cell = (
+                        ticker,
+                        col,
+                        getattr(worst_dt, "isoformat", lambda: str(worst_dt))(),
+                    )
+
+    return ParityReport(
+        only_in_a=frozenset(keys_a - keys_b),
+        only_in_b=frozenset(keys_b - keys_a),
+        common=frozenset(common),
+        rowcount_deltas=rowcount_deltas,
+        max_abs_value_delta=max_abs,
+        worst_cell=worst_cell,
+        n_cells_over_epsilon=n_over,
+        n_cells_compared=n_compared,
+        epsilon=epsilon,
+        value_cols=cols,
+        require_ticker_match=require_ticker_match,
+        require_rowcount_match=require_rowcount_match,
+    )

alpha_engine_lib/secrets.py

@@ -0,0 +1,186 @@
+"""
+Per-key SSM-backed secret fetcher for Alpha Engine modules.
+
+This module is the consolidation point for secret resolution across all six
+alpha-engine repos. Before this, each repo duplicated some variant of the
+``ssm_secrets.py`` bulk-load-into-os.environ pattern (alpha-engine-data has
+the canonical one). Going forward, callers do::
+
+    from alpha_engine_lib.secrets import get_secret
+
+    api_key = get_secret("ANTHROPIC_API_KEY")              # required → raises if absent
+    opt_key = get_secret("DEBUG_TRACE_KEY", required=False)  # optional → returns None
+
+**Resolution order** (first hit wins):
+
+1. Per-process cache (populated on first read; thread-safe).
+2. ``ALPHA_ENGINE_SECRETS_SOURCE`` env-var toggle:
+
+   - ``env`` → ``os.environ[name]`` only (local-dev escape hatch — never hit SSM)
+   - ``ssm`` → SSM only (production strictness — no silent env fallback)
+   - unset / ``auto`` / anything else → SSM first, ``os.environ`` fallback
+
+3. SSM at ``/alpha-engine/{name}`` (under :data:`SSM_PREFIX`).
+4. ``os.environ[name]`` (the fallback in ``auto`` mode).
+5. ``default`` arg if provided.
+6. :exc:`SecretNotFoundError` if ``required=True``; else ``None``.
+
+**Caching design.** Per-process dict keyed on secret name. The SSM round-trip
+happens at most once per name per process; subsequent calls hit the dict.
+Lambda cold-starts pay the round-trip once; warm invocations reuse the cache.
+:func:`clear_cache` is exposed for tests.
+
+**SSM-unavailable latch.** If the first SSM call fails (no boto3, no creds,
+network error), latch ``_ssm_unavailable = True`` and skip SSM for the rest of
+the process. Avoids repeated multi-second timeouts in local dev. Reset via
+:func:`clear_cache` if a test needs to re-probe SSM.
+
+**Migration arc**: ``alpha-engine-config/private-docs/ROADMAP.md`` line ~2780
+(Deprecate ``.env`` entirely). Plan doc:
+``alpha-engine-docs/private/env-to-ssm-260512.md``.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+from typing import Final
+
+logger = logging.getLogger(__name__)
+
+SSM_PREFIX: Final[str] = "/alpha-engine/"
+SOURCE_TOGGLE_ENV: Final[str] = "ALPHA_ENGINE_SECRETS_SOURCE"
+
+_cache: dict[str, str] = {}
+_cache_lock = threading.Lock()
+_ssm_unavailable = False
+
+
+class SecretNotFoundError(LookupError):
+    """Raised when a required secret is missing from both SSM and the environment."""
+
+
+def get_secret(
+    name: str,
+    *,
+    required: bool = True,
+    default: str | None = None,
+) -> str | None:
+    """Fetch a secret by ``name`` from SSM with environment fallback.
+
+    See module docstring for the full resolution order. The lookup is
+    per-process cached; the first call per process pays the SSM round-trip,
+    subsequent calls hit the in-memory dict.
+
+    :param name: Secret name (no prefix). E.g. ``"POLYGON_API_KEY"``.
+    :param required: If ``True`` (default), raise :exc:`SecretNotFoundError`
+        when the secret is absent. If ``False``, return ``default`` (or
+        ``None`` if ``default`` is unset).
+    :param default: Value to return if the secret is absent and
+        ``required=False``. Ignored when ``required=True``.
+    :raises SecretNotFoundError: When ``required=True`` and the secret is
+        absent from cache, SSM, and ``os.environ``.
+    :raises ValueError: When ``name`` is empty or contains a forward slash.
+    """
+    if not name:
+        raise ValueError("secret name must be non-empty")
+    if "/" in name:
+        raise ValueError(
+            f"secret name must not contain '/': got {name!r} "
+            f"(the SSM_PREFIX is added automatically)"
+        )
+
+    with _cache_lock:
+        cached = _cache.get(name)
+    if cached is not None:
+        return cached
+
+    source = os.environ.get(SOURCE_TOGGLE_ENV, "auto").lower()
+    if source not in ("auto", "env", "ssm"):
+        logger.warning(
+            "unknown %s=%r — falling back to 'auto'", SOURCE_TOGGLE_ENV, source
+        )
+        source = "auto"
+
+    value: str | None = None
+
+    if source in ("auto", "ssm"):
+        value = _fetch_from_ssm(name)
+
+    if value is None and source in ("auto", "env"):
+        value = os.environ.get(name)
+
+    if value is None:
+        if default is not None:
+            return default
+        if required:
+            raise SecretNotFoundError(
+                f"secret {name!r} not found in cache, SSM ({SSM_PREFIX}{name}), "
+                f"or environment (source={source!r})"
+            )
+        return None
+
+    with _cache_lock:
+        _cache[name] = value
+    return value
+
+
+def clear_cache() -> None:
+    """Clear the per-process cache and re-arm SSM probing.
+
+    Mostly for tests — production code should not need to call this. Resets
+    both the secret cache and the ``_ssm_unavailable`` latch.
+    """
+    global _ssm_unavailable
+    with _cache_lock:
+        _cache.clear()
+        _ssm_unavailable = False
+
+
+def _fetch_from_ssm(name: str) -> str | None:
+    """Single-key SSM read. Returns ``None`` on miss or unavailability."""
+    global _ssm_unavailable
+    if _ssm_unavailable:
+        return None
+
+    try:
+        import boto3
+        from botocore.exceptions import BotoCoreError, ClientError
+    except ImportError:
+        logger.debug("boto3 not installed — skipping SSM for %s", name)
+        _ssm_unavailable = True
+        return None
+
+    region = os.environ.get("AWS_REGION") or os.environ.get(
+        "AWS_DEFAULT_REGION", "us-east-1"
+    )
+    try:
+        client = boto3.client("ssm", region_name=region)
+        resp = client.get_parameter(
+            Name=f"{SSM_PREFIX}{name}",
+            WithDecryption=True,
+        )
+        return resp["Parameter"]["Value"]
+    except ClientError as e:
+        code = e.response.get("Error", {}).get("Code", "")
+        if code == "ParameterNotFound":
+            # Genuine miss — not an SSM-availability problem. Fall through
+            # to env without latching, since other secrets may resolve fine.
+            logger.debug("SSM miss for %s (ParameterNotFound)", name)
+            return None
+        logger.warning(
+            "SSM read for %s failed (%s) — latching unavailable for this process",
+            name,
+            code or "unknown",
+        )
+        _ssm_unavailable = True
+        return None
+    except BotoCoreError as e:
+        logger.warning(
+            "SSM read for %s failed (%s) — latching unavailable for this process",
+            name,
+            type(e).__name__,
+        )
+        _ssm_unavailable = True
+        return None

alpha_engine_lib/sources/__init__.py

@@ -0,0 +1,35 @@
+"""Shared data-source contracts — Pydantic shapes + Protocols.
+
+This package defines the canonical normalized shapes (``NewsArticle``,
+``AnalystSnapshot``, ``FilingDocument``) and the adapter Protocols
+(``NewsSource``, ``AnalystSource``, ``FilingSource``) that both
+producers (alpha-engine-data) and consumers (alpha-engine-research,
+alpha-engine-backtester) consume.
+
+**Architectural pattern:** Lib defines the contract; producers
+(alpha-engine-data) implement concrete adapters; consumers
+(alpha-engine-research) read records produced by them via S3 / RAG
+retrieval and never import adapter classes directly.
+
+See the institutional data-revamp plan doc at
+``~/Development/alpha-engine-docs/private/data-revamp-260513.md`` for
+full context. PR α — lib (this); PR β — data (adapter implementations).
+"""
+
+from alpha_engine_lib.sources.protocols import (
+    AnalystSnapshot,
+    AnalystSource,
+    FilingDocument,
+    FilingSource,
+    NewsArticle,
+    NewsSource,
+)
+
+__all__ = [
+    "NewsArticle",
+    "AnalystSnapshot",
+    "FilingDocument",
+    "NewsSource",
+    "AnalystSource",
+    "FilingSource",
+]

alpha_engine_lib/sources/protocols.py

@@ -0,0 +1,227 @@
+"""Source-substrate Protocols + normalized Pydantic shapes.
+
+Wave 1 PR α of the institutional data revamp (see
+``~/Development/alpha-engine-docs/private/data-revamp-260513.md``). Each
+data slot (news, filings, analyst, alt data) becomes a Protocol with
+multiple adapters implementing it; concrete adapters live in
+alpha-engine-data (producer-side) and consumers (alpha-engine-research)
+never import them directly — they read producer outputs via S3 + the
+shared RAG retrieval API.
+
+Why Protocols over ABCs:
+
+- Structural subtyping — third-party SDK wrappers can satisfy without
+  inheriting from our base class.
+- Static type-checking via ``runtime_checkable`` for explicit gating
+  in the aggregator.
+- No vtable overhead in hot loops.
+
+Why Pydantic shapes (not raw dicts):
+
+- Cross-vendor schema normalization is brittle. Pydantic gives us a
+  single canonical shape with validation at the adapter boundary —
+  adapter bugs surface as ``ValidationError``, not as silent
+  ``KeyError`` in downstream NLP three layers deeper.
+- ``extra='forbid'`` ensures vendor schema drift surfaces immediately
+  instead of leaking unmapped fields through the pipeline.
+- ``frozen=True`` so records are hashable + safe to share across
+  threads / fan-out workers without defensive copies.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# ── Normalized shapes ──────────────────────────────────────────────────
+
+
+class NewsArticle(BaseModel):
+    """One news article, normalized across all vendors.
+
+    The canonical key for cross-vendor dedup is the composite
+    (normalized title, URL host+path hash). Different vendors syndicate
+    the same wire story; the aggregator's dedup catches both the URL-
+    collapse case (different querystrings) and the
+    title-paraphrase-on-same-URL case.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    tickers: tuple[str, ...] = Field(
+        description="Tickers this article concerns. Multi-ticker articles "
+                    "(e.g. sector pieces) are kept as a single record with "
+                    "a multi-element tuple; the aggregator's ticker-union "
+                    "logic merges variants. Adapter's choice: emit once per "
+                    "(article, ticker) or once with the full set."
+    )
+    title: str
+    body_excerpt: str = Field(
+        description="Lead paragraph or summary. Full-text body lives in the "
+                    "RAG corpus chunk store, not in this struct — chunked at "
+                    "embedding time by the ingest pipeline."
+    )
+    url: str
+    published_at: datetime = Field(
+        description="UTC publish time. Vendor wall-clock; ingest-time is "
+                    "in `fetched_at`."
+    )
+    source: str = Field(
+        description="Vendor slug: 'polygon', 'gdelt', 'yahoo_rss', "
+                    "'edgar_press', 'benzinga' (paid), 'bloomberg' (paid), "
+                    "'ravenpack' (paid). Joins onto the trust-weight config "
+                    "downstream."
+    )
+    vendor_article_id: str | None = Field(
+        default=None,
+        description="Vendor-native unique ID for cross-reference back to "
+                    "the source system (Polygon `id`, GDELT `GKGRECORDID`, "
+                    "etc.). Used by ingest-side idempotency checks.",
+    )
+    fetched_at: datetime = Field(
+        description="When this adapter pulled the article (UTC). For "
+                    "freshness audit + cache-age computation."
+    )
+    headline_authors: tuple[str, ...] | None = Field(
+        default=None,
+        description="Bylines if available. None if the source doesn't "
+                    "expose authors (e.g. wire feeds).",
+    )
+    tags: tuple[str, ...] = Field(
+        default_factory=tuple,
+        description="Vendor-supplied topic / event tags. GDELT emits "
+                    "structured event codes; Polygon emits keywords; "
+                    "Benzinga emits Channels. Used as a soft signal for "
+                    "downstream event-flag extraction.",
+    )
+
+
+class AnalystSnapshot(BaseModel):
+    """One vendor's analyst consensus snapshot for one ticker at one
+    point in time.
+
+    Time-series of these in S3 drives self-derived revisions tracking
+    (see ``alpha-engine-data/data/derived/revisions.py``, PR C).
+    Adapter is responsible for vendor-string normalization at the
+    boundary — downstream consumers see the canonical 5-class
+    consensus_rating ladder.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    ticker: str
+    source: str
+    fetched_at: datetime
+    consensus_rating: str | None = Field(
+        default=None,
+        description="Categorical: 'strongBuy' | 'buy' | 'hold' | 'sell' | "
+                    "'strongSell'. Vendor strings normalized at adapter "
+                    "boundary.",
+    )
+    mean_target: float | None = Field(
+        default=None, description="Mean price target (USD)."
+    )
+    median_target: float | None = Field(
+        default=None, description="Median price target if vendor exposes it."
+    )
+    num_analysts: int | None = Field(
+        default=None, description="Number of contributing analysts."
+    )
+    rating_changes_30d: tuple[dict, ...] = Field(
+        default_factory=tuple,
+        description="Recent upgrades/downgrades. Each entry: "
+                    "{analyst, firm, action, prior_rating, new_rating, "
+                    "date}. Used by downstream NLP/event-flag extraction.",
+    )
+
+
+class FilingDocument(BaseModel):
+    """One filing document. Filings substrate (PR B). Pinned here so
+    PR α can reference the shape from Protocols without forward refs."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    ticker: str
+    form_type: str = Field(
+        description="'10-K' | '10-Q' | '8-K' | '14A' | 'DEF' | 'S-1' | "
+                    "'S-4' | '13D' | '13G' | '13F' | 'Form 4' | etc."
+    )
+    filed_date: datetime
+    accession_number: str = Field(
+        description="EDGAR accession (e.g. '0000320193-25-000001'). "
+                    "Canonical key for filing dedup + RAG idempotency check."
+    )
+    title: str | None = None
+    url: str
+    source: str = "edgar"
+    fetched_at: datetime
+    body_excerpt: str = Field(
+        description="Lead snippet. Full body goes to the RAG corpus, "
+                    "chunked at ingest time."
+    )
+
+
+# ── Protocols ──────────────────────────────────────────────────────────
+
+
+@runtime_checkable
+class NewsSource(Protocol):
+    """News adapter contract. Adapters are vendor-specific transports
+    that produce normalized :class:`NewsArticle` records.
+
+    Adapters MUST:
+
+    - Be safely callable from concurrent contexts (own their HTTP client +
+      rate-limiter; no shared mutable state).
+    - Return an empty list (never raise) on transient vendor failures.
+      Re-raise only on auth failures, contract-breaking schema drift, or
+      configuration errors — those fail loud.
+    - Normalize wall-clock timestamps to UTC.
+    - Stamp ``fetched_at`` on every returned article.
+    """
+
+    name: str  # vendor slug — joins onto trust-weight config
+
+    def fetch(
+        self,
+        tickers: list[str],
+        *,
+        hours: int = 48,
+    ) -> list[NewsArticle]: ...
+
+
+@runtime_checkable
+class AnalystSource(Protocol):
+    """Analyst data adapter contract. PR C.
+
+    Returns ``None`` for tickers the vendor doesn't cover (e.g. micro-
+    caps absent from FMP); empty fields within a returned snapshot
+    indicate vendor-side coverage gaps that are typed in the shape.
+    """
+
+    name: str
+
+    def fetch(self, ticker: str) -> AnalystSnapshot | None: ...
+
+
+@runtime_checkable
+class FilingSource(Protocol):
+    """Filings adapter contract. PR B.
+
+    ``form_types`` filter is advisory — adapters that don't support
+    form-type pre-filtering at the vendor layer return all forms and
+    let downstream filter.
+    """
+
+    name: str
+
+    def fetch(
+        self,
+        tickers: list[str],
+        *,
+        form_types: list[str] | None = None,
+        days: int = 7,
+    ) -> list[FilingDocument]: ...