ml4t-data 0.1.0__py3-none-any.whl

ml4t/data/AGENTS.md

@@ -0,0 +1,46 @@
+# ml4t.data - Package Index
+
+## Core Modules
+
+| File | Lines | Purpose |
+|------|-------|---------|
+| data_manager.py | 564 | Main orchestration API |
+| update_manager.py | 829 | Incremental update system |
+| universe.py | 987 | Asset universe management |
+| provider_updater.py | 449 | Provider coordination |
+
+## Subpackages
+
+| Directory | Lines | Purpose |
+|-----------|-------|---------|
+| providers/ | 14k | 20 live provider adapters + synthetic/testing providers |
+| storage/ | 4.6k | Hive-partitioned backends + profiling |
+| futures/ | 4.6k | Databento futures downloader |
+| etfs/ | 600 | ETFDataManager (Yahoo Finance) |
+| crypto/ | 420 | CryptoDataManager (Binance Public) |
+| core/ | 1k | Models, schemas, config |
+| utils/ | 1.6k | Rate limiting, gaps, retry |
+| assets/ | 1.3k | Asset class definitions |
+| anomaly/ | 1k | Data quality detection |
+| cot/ | 1k | CFTC COT data |
+| validation/ | 1.2k | OHLC validation |
+| sessions/ | 462 | Session assignment |
+| macro/ | 455 | Macro data downloader |
+| calendar/ | 359 | Trading calendars |
+| export/ | 231 | CSV/JSON/Excel export |
+
+## Book Data Managers
+
+Simplified managers for ML4T book readers with built-in profiling:
+
+| Manager | Asset | Source | Profiling |
+|---------|-------|--------|-----------|
+| `ETFDataManager` | ETFs | Yahoo Finance | `generate_profile()` |
+| `CryptoDataManager` | Crypto | Binance Public | `generate_profile()` |
+| `FuturesDataManager` | Futures | Databento | `generate_profile(product)` |
+
+All managers inherit from `ProfileMixin` for on-demand column statistics.
+
+## Key
+
+`DataManager`, `UpdateManager`, `HiveStorage`, `ProfileMixin`, `get_provider()`

ml4t/data/__init__.py

@@ -0,0 +1,54 @@
+"""ML4T Data - Modern financial data management library with unified provider interface."""
+
+try:
+    from ml4t.data._version import __version__
+except ImportError:
+    __version__ = "0.0.0.dev0"
+__author__ = "ML4T Team"
+__email__ = "info@ml4trading.io"
+
+# Contract specifications (always available - no external dependencies)
+# Asset classes
+from ml4t.data.assets.asset_class import AssetClass
+from ml4t.data.assets.contracts import (
+    FUTURES_REGISTRY,
+    ContractSpec,
+    get_contract_spec,
+    load_contract_specs,
+    register_contract_spec,
+)
+from ml4t.data.core.exceptions import ProviderRoutingError
+
+__all__ = [
+    # Contract specifications
+    "ContractSpec",
+    "FUTURES_REGISTRY",
+    "get_contract_spec",
+    "load_contract_specs",
+    "register_contract_spec",
+    "AssetClass",
+    "ProviderRoutingError",
+]
+
+# Core imports (may have additional dependencies)
+try:
+    from ml4t.data.core.config import Config  # noqa: F401
+    from ml4t.data.data_manager import DataManager  # noqa: F401
+    from ml4t.data.providers.base import BaseProvider  # noqa: F401
+
+    __all__.extend(["BaseProvider", "Config", "DataManager"])
+except ImportError:
+    # Try to import what's available
+    try:
+        from ml4t.data.data_manager import DataManager  # noqa: F401
+
+        __all__.append("DataManager")
+    except ImportError:
+        pass
+
+    try:
+        from ml4t.data.providers.base import BaseProvider  # noqa: F401
+
+        __all__.append("BaseProvider")
+    except ImportError:
+        pass

ml4t/data/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

ml4t/data/adjustments/__init__.py

@@ -0,0 +1,23 @@
+"""Corporate actions and price adjustment utilities.
+
+The canonical convention treats a split ratio as new shares per old share on
+the event date. Raw values are retained, and adjusted values use the share
+basis of the latest observation.
+
+Example:
+    >>> from ml4t.data.adjustments import apply_corporate_actions
+    >>> adjusted_prices = apply_corporate_actions(unadjusted_df)
+    >>> adjusted_prices.select("close", "adj_close", "price_adjustment_factor")
+"""
+
+from .core import (
+    apply_corporate_actions,
+    apply_dividends,
+    apply_splits,
+)
+
+__all__ = [
+    "apply_splits",
+    "apply_dividends",
+    "apply_corporate_actions",
+]

ml4t/data/adjustments/core.py

@@ -0,0 +1,252 @@
+"""Corporate-action adjustment functions."""
+
+import polars as pl
+
+
+def _validate_adjustment_inputs(
+    df: pl.DataFrame,
+    *,
+    split_col: str,
+    dividend_col: str | None,
+    price_cols: list[str],
+    volume_col: str | None,
+) -> None:
+    required = {"date", split_col, *price_cols}
+    if dividend_col is not None:
+        required.update(("close", dividend_col))
+    missing = sorted(required.difference(df.columns))
+    if missing:
+        raise ValueError(f"Missing required adjustment columns: {', '.join(missing)}")
+    if df.get_column("date").null_count():
+        raise ValueError("Corporate-action dates cannot contain null values")
+    if df.get_column("date").n_unique() != df.height:
+        raise ValueError("Corporate-action input must contain exactly one row per date")
+
+    finite_columns = [split_col, *price_cols]
+    if dividend_col is not None:
+        finite_columns.extend(("close", dividend_col))
+    if volume_col is not None and volume_col in df.columns:
+        finite_columns.append(volume_col)
+    for column in dict.fromkeys(finite_columns):
+        invalid = df.filter(pl.col(column).is_null() | ~pl.col(column).is_finite())
+        if not invalid.is_empty():
+            raise ValueError(f"Adjustment column '{column}' must contain finite values")
+
+    if not df.filter(pl.col(split_col) <= 0).is_empty():
+        raise ValueError(f"Split ratios in '{split_col}' must be positive")
+    if dividend_col is not None and not df.filter(pl.col("close") <= 0).is_empty():
+        raise ValueError("Close prices must be positive")
+    if dividend_col is not None and not df.filter(pl.col(dividend_col) < 0).is_empty():
+        raise ValueError(f"Dividends in '{dividend_col}' cannot be negative")
+    if volume_col is not None and volume_col in df.columns:
+        if not df.filter(pl.col(volume_col) < 0).is_empty():
+            raise ValueError(f"Volume in '{volume_col}' cannot be negative")
+
+
+def _future_cumulative_product(column: str) -> pl.Expr:
+    return pl.col(column).shift(-1, fill_value=1.0).reverse().cum_prod().reverse()
+
+
+def _apply_canonical_adjustments(
+    prices: pl.DataFrame,
+    *,
+    split_col: str,
+    dividend_col: str | None,
+    price_cols: list[str],
+    volume_col: str | None,
+) -> pl.DataFrame:
+    _validate_adjustment_inputs(
+        prices,
+        split_col=split_col,
+        dividend_col=dividend_col,
+        price_cols=price_cols,
+        volume_col=volume_col,
+    )
+    df = prices.sort("date").clone()
+
+    price_event_factor = (
+        pl.col("close")
+        / (pl.col(split_col).cast(pl.Float64) * (pl.col("close") + pl.col(dividend_col)))
+        if dividend_col is not None
+        else 1.0 / pl.col(split_col).cast(pl.Float64)
+    )
+    df = df.with_columns(
+        price_event_factor.alias("_price_event_factor"),
+        pl.col(split_col).cast(pl.Float64).alias("_volume_event_factor"),
+    ).with_columns(
+        _future_cumulative_product("_price_event_factor").alias("price_adjustment_factor"),
+        _future_cumulative_product("_volume_event_factor").alias("volume_adjustment_factor"),
+    )
+
+    adjustments = [
+        (pl.col(column) * pl.col("price_adjustment_factor")).alias(f"adj_{column}")
+        for column in price_cols
+    ]
+    if volume_col is not None and volume_col in df.columns:
+        adjustments.append(
+            (pl.col(volume_col) * pl.col("volume_adjustment_factor")).alias(f"adj_{volume_col}")
+        )
+    return df.with_columns(adjustments).drop("_price_event_factor", "_volume_event_factor")
+
+
+def apply_corporate_actions(
+    prices: pl.DataFrame,
+    split_col: str = "split_ratio",
+    dividend_col: str = "ex-dividend",
+    price_cols: list[str] | None = None,
+    volume_col: str | None = "volume",
+) -> pl.DataFrame:
+    """Adjust prices and volume to the share basis of the latest observation.
+
+    A split ratio on date ``t`` is new shares per old share and applies between
+    the preceding observation and ``t``. The row on ``t`` is already on the new
+    share basis. Earlier prices are divided by subsequent split ratios and
+    earlier volume is multiplied by them. A dividend is cash per post-event
+    share on its ex-date. Adjusted close-to-close returns include that cash.
+    The dividend factor uses the ex-date close. This differs from data vendors
+    that discount dividends using the prior close.
+
+    Args:
+        prices: DataFrame with date-sorted prices and corporate action data
+        split_col: Column with split ratios (default: 'split_ratio')
+        dividend_col: Column with dividend amounts (default: 'ex-dividend')
+        price_cols: Price columns to adjust (default: ['open', 'high', 'low', 'close'])
+        volume_col: Volume column to adjust (default: 'volume')
+
+    Returns:
+        DataFrame retaining raw columns and adding adjusted columns plus explicit
+        price and volume adjustment factors
+    """
+    if price_cols is None:
+        price_cols = [
+            column for column in ("open", "high", "low", "close") if column in prices.columns
+        ]
+    return _apply_canonical_adjustments(
+        prices,
+        split_col=split_col,
+        dividend_col=dividend_col,
+        price_cols=price_cols,
+        volume_col=volume_col,
+    )
+
+
+def apply_splits(
+    prices: pl.DataFrame,
+    split_col: str = "split_ratio",
+    price_cols: list[str] | None = None,
+    volume_col: str | None = "volume",
+) -> pl.DataFrame:
+    """Apply the canonical split-only adjustment convention.
+
+    Args:
+        prices: DataFrame with date-sorted prices and split_ratio column
+        split_col: Name of column containing split ratios (default: 'split_ratio')
+        price_cols: List of price columns to adjust (default: ['open', 'high', 'low', 'close'])
+        volume_col: Volume column name to adjust
+
+    Returns:
+        DataFrame retaining raw values and adding adjusted values and factors
+    """
+    if price_cols is None:
+        price_cols = [
+            column for column in ("open", "high", "low", "close") if column in prices.columns
+        ]
+    return _apply_canonical_adjustments(
+        prices,
+        split_col=split_col,
+        dividend_col=None,
+        price_cols=price_cols,
+        volume_col=volume_col,
+    )
+
+
+def apply_dividends(
+    prices: pl.DataFrame,
+    dividend_col: str = "ex-dividend",
+    price_cols: list[str] | None = None,
+    close_col: str = "adj_close",
+) -> pl.DataFrame:
+    """Apply dividend-only adjustments that preserve total close-to-close returns.
+
+    A dividend on date ``t`` is cash per share paid between the preceding
+    observation and ``t``. The event row remains unchanged. This adapter is for
+    data whose split adjustments have already been handled consistently. When
+    split factors are present, dividends are converted to the latest share basis
+    and the dividend factor is composed with the existing price factor.
+
+    Args:
+        prices: DataFrame with date-sorted prices and ex-dividend column
+        dividend_col: Name of column containing dividend amounts
+        price_cols: List of price columns to adjust
+        close_col: Column to use for dividend factor calculation (default: 'adj_close')
+
+    Returns:
+        DataFrame with adjusted price columns and an explicit adjustment factor
+    """
+    if price_cols is None:
+        price_cols = [
+            column
+            for column in ("adj_open", "adj_high", "adj_low", "adj_close")
+            if column in prices.columns
+        ]
+
+    required = {"date", dividend_col, close_col, *price_cols}
+    missing = sorted(required.difference(prices.columns))
+    if missing:
+        raise ValueError(f"Missing required dividend columns: {', '.join(missing)}")
+    df = prices.sort("date").clone()
+    if df.get_column("date").null_count():
+        raise ValueError("Dividend dates cannot contain null values")
+    if df.get_column("date").n_unique() != df.height:
+        raise ValueError("Dividend input must contain exactly one row per date")
+    for column in (dividend_col, close_col, *price_cols):
+        if not df.filter(pl.col(column).is_null() | ~pl.col(column).is_finite()).is_empty():
+            raise ValueError(f"Dividend adjustment column '{column}' must contain finite values")
+    if not df.filter(pl.col(dividend_col) < 0).is_empty():
+        raise ValueError(f"Dividends in '{dividend_col}' cannot be negative")
+    if not df.filter(pl.col(close_col) <= 0).is_empty():
+        raise ValueError(f"Close prices in '{close_col}' must be positive")
+
+    for factor_column in ("price_adjustment_factor", "volume_adjustment_factor"):
+        if (
+            factor_column in df.columns
+            and not df.filter(
+                pl.col(factor_column).is_null()
+                | ~pl.col(factor_column).is_finite()
+                | (pl.col(factor_column) <= 0)
+            ).is_empty()
+        ):
+            raise ValueError(f"Adjustment factor '{factor_column}' must be finite and positive")
+
+    rebased_dividend = pl.col(dividend_col)
+    if "volume_adjustment_factor" in df.columns:
+        rebased_dividend = rebased_dividend / pl.col("volume_adjustment_factor")
+    existing_factor = (
+        pl.col("price_adjustment_factor")
+        if "price_adjustment_factor" in df.columns
+        else pl.lit(1.0)
+    )
+
+    return (
+        df.with_columns(rebased_dividend.alias("_rebased_dividend"))
+        .with_columns(
+            (pl.col(close_col) / (pl.col(close_col) + pl.col("_rebased_dividend"))).alias(
+                "_dividend_event_factor"
+            )
+        )
+        .with_columns(
+            _future_cumulative_product("_dividend_event_factor").alias(
+                "_dividend_adjustment_factor"
+            )
+        )
+        .with_columns(
+            *[
+                (pl.col(column) * pl.col("_dividend_adjustment_factor")).alias(column)
+                for column in price_cols
+            ],
+            (existing_factor * pl.col("_dividend_adjustment_factor")).alias(
+                "price_adjustment_factor"
+            ),
+        )
+        .drop("_rebased_dividend", "_dividend_event_factor", "_dividend_adjustment_factor")
+    )

ml4t/data/anomaly/AGENTS.md

@@ -0,0 +1,16 @@
+# anomaly/ - 1k Lines
+
+Data quality and anomaly detection.
+
+## Functions
+
+| Function | Purpose |
+|----------|---------|
+| detect_anomalies | Find data anomalies |
+| validate_ohlc | OHLC consistency |
+| check_gaps | Missing data detection |
+| flag_outliers | Statistical outliers |
+
+## Key
+
+`AnomalyDetector`, `detect_anomalies()`, `DataQualityReport`

ml4t/data/anomaly/__init__.py

@@ -0,0 +1,23 @@
+"""Anomaly detection for financial data."""
+
+from ml4t.data.anomaly.base import Anomaly, AnomalyDetector, AnomalyReport, AnomalySeverity
+from ml4t.data.anomaly.config import AnomalyConfig, DetectorConfig
+from ml4t.data.anomaly.detectors import (
+    PriceStalenessDetector,
+    ReturnOutlierDetector,
+    VolumeSpikeDetector,
+)
+from ml4t.data.anomaly.manager import AnomalyManager
+
+__all__ = [
+    "Anomaly",
+    "AnomalyConfig",
+    "AnomalyDetector",
+    "AnomalyManager",
+    "AnomalyReport",
+    "AnomalySeverity",
+    "DetectorConfig",
+    "PriceStalenessDetector",
+    "ReturnOutlierDetector",
+    "VolumeSpikeDetector",
+]

ml4t/data/anomaly/base.py

@@ -0,0 +1,142 @@
+"""Base classes for anomaly detection."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+import polars as pl
+from pydantic import BaseModel, Field
+
+
+class AnomalySeverity(StrEnum):
+    """Anomaly severity levels."""
+
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+    CRITICAL = "critical"
+
+
+class AnomalyType(StrEnum):
+    """Types of anomalies."""
+
+    RETURN_OUTLIER = "return_outlier"
+    VOLUME_SPIKE = "volume_spike"
+    PRICE_STALE = "price_stale"
+    DATA_GAP = "data_gap"
+    PRICE_SPIKE = "price_spike"
+    ZERO_VOLUME = "zero_volume"
+    NEGATIVE_PRICE = "negative_price"
+
+
+class Anomaly(BaseModel):
+    """Represents a detected anomaly."""
+
+    timestamp: datetime = Field(description="When the anomaly occurred")
+    symbol: str = Field(description="Symbol with the anomaly")
+    type: AnomalyType = Field(description="Type of anomaly")
+    severity: AnomalySeverity = Field(description="Severity level")
+    value: float = Field(description="The anomalous value")
+    expected_range: tuple[float, float] | None = Field(
+        default=None, description="Expected value range"
+    )
+    threshold: float | None = Field(default=None, description="Detection threshold used")
+    message: str = Field(description="Human-readable description")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional context")
+
+    def __str__(self) -> str:
+        """String representation."""
+        return f"[{self.severity.value.upper()}] {self.symbol} @ {self.timestamp}: {self.message}"
+
+
+class AnomalyDetector(ABC):
+    """Abstract base class for anomaly detectors."""
+
+    def __init__(self, enabled: bool = True):
+        """
+        Initialize detector.
+
+        Args:
+            enabled: Whether this detector is enabled
+        """
+        self.enabled = enabled
+
+    @abstractmethod
+    def detect(self, df: pl.DataFrame, symbol: str) -> list[Anomaly]:
+        """
+        Detect anomalies in the data.
+
+        Args:
+            df: DataFrame with OHLCV data
+            symbol: Symbol being analyzed
+
+        Returns:
+            List of detected anomalies
+        """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Return detector name."""
+
+    def is_enabled(self) -> bool:
+        """Check if detector is enabled."""
+        return self.enabled
+
+
+class AnomalyReport(BaseModel):
+    """Anomaly detection report."""
+
+    symbol: str = Field(description="Symbol analyzed")
+    start_date: datetime = Field(description="Start of analysis period")
+    end_date: datetime = Field(description="End of analysis period")
+    total_rows: int = Field(description="Total data points analyzed")
+    anomalies: list[Anomaly] = Field(default_factory=list, description="Detected anomalies")
+    summary: dict[str, int] = Field(default_factory=dict, description="Summary statistics")
+    detectors_used: list[str] = Field(default_factory=list, description="Detectors applied")
+
+    def add_anomaly(self, anomaly: Anomaly) -> None:
+        """Add an anomaly to the report."""
+        self.anomalies.append(anomaly)
+
+        # Update summary
+        severity_key = f"{anomaly.severity.value}_count"
+        self.summary[severity_key] = self.summary.get(severity_key, 0) + 1
+
+        type_key = f"{anomaly.type.value}_count"
+        self.summary[type_key] = self.summary.get(type_key, 0) + 1
+
+    def get_critical_anomalies(self) -> list[Anomaly]:
+        """Get only critical anomalies."""
+        return [a for a in self.anomalies if a.severity == AnomalySeverity.CRITICAL]
+
+    def get_by_type(self, anomaly_type: AnomalyType) -> list[Anomaly]:
+        """Get anomalies of specific type."""
+        return [a for a in self.anomalies if a.type == anomaly_type]
+
+    def has_critical_issues(self) -> bool:
+        """Check if there are any critical issues."""
+        return any(a.severity == AnomalySeverity.CRITICAL for a in self.anomalies)
+
+    def to_dataframe(self) -> pl.DataFrame:
+        """Convert anomalies to DataFrame for analysis."""
+        if not self.anomalies:
+            return pl.DataFrame()
+
+        records = []
+        for anomaly in self.anomalies:
+            records.append(
+                {
+                    "timestamp": anomaly.timestamp,
+                    "symbol": anomaly.symbol,
+                    "type": anomaly.type.value,
+                    "severity": anomaly.severity.value,
+                    "value": anomaly.value,
+                    "message": anomaly.message,
+                }
+            )
+
+        return pl.DataFrame(records)