ml4t-data 0.2.0__tar.gz → 0.3.0__tar.gz

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ml4t-data
-Version: 0.2.0
+Version: 0.3.0
 Summary: Market data acquisition, storage, and update workflows for machine learning for trading.
 Project-URL: Homepage, https://www.ml4trading.io/
 Project-URL: Documentation, https://www.ml4trading.io/docs/data/
@@ -29,15 +29,17 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Typing :: Typed
 Requires-Python: <3.15,>=3.12
 Requires-Dist: aiofiles>=23.0.0
+Requires-Dist: certifi>=2026.10.19
 Requires-Dist: click>=8.0.0
 Requires-Dist: filelock>=3.19.1
 Requires-Dist: httpx>=0.25.0
+Requires-Dist: ml4t-specs>=0.2.0
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: openpyxl>=3.1.5
-Requires-Dist: pandas-market-calendars>=4.3.0
+Requires-Dist: pandas-market-calendars>=5.4.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: platformdirs>=4.0.0
-Requires-Dist: polars>=0.20.0
+Requires-Dist: polars>=1.36.1
 Requires-Dist: pybreaker>=1.0.0
 Requires-Dist: pydantic-settings>=2.0.0
 Requires-Dist: pydantic<3,>=2.12
@@ -68,14 +70,14 @@ Requires-Dist: requests>=2.32.0; extra == 'all'
 Requires-Dist: ruff>=0.1.0; extra == 'all'
 Requires-Dist: ty; extra == 'all'
 Requires-Dist: xlsxwriter>=3.1.0; extra == 'all'
-Requires-Dist: yfinance>=0.2.0; extra == 'all'
+Requires-Dist: yfinance>=1.5.2; extra == 'all'
 Provides-Extra: all-providers
 Requires-Dist: cot-reports>=0.1.0; extra == 'all-providers'
 Requires-Dist: databento>=0.38.0; extra == 'all-providers'
 Requires-Dist: oandapyv20>=0.7.0; extra == 'all-providers'
 Requires-Dist: pyarrow>=14.0.0; extra == 'all-providers'
 Requires-Dist: requests>=2.32.0; extra == 'all-providers'
-Requires-Dist: yfinance>=0.2.0; extra == 'all-providers'
+Requires-Dist: yfinance>=1.5.2; extra == 'all-providers'
 Provides-Extra: cot
 Requires-Dist: cot-reports>=0.1.0; extra == 'cot'
 Provides-Extra: databento
@@ -104,7 +106,7 @@ Provides-Extra: oanda
 Requires-Dist: oandapyv20>=0.7.0; extra == 'oanda'
 Requires-Dist: requests>=2.32.0; extra == 'oanda'
 Provides-Extra: yahoo
-Requires-Dist: yfinance>=0.2.0; extra == 'yahoo'
+Requires-Dist: yfinance>=1.5.2; extra == 'yahoo'
 Description-Content-Type: text/markdown
 
 # ml4t-data
@@ -179,7 +181,7 @@ uv run pre-commit install
 uv run ruff check src/ tests/
 uv run ruff format --check src/ tests/
 uv run ty check
-uv run pytest tests/ -q
+uv run pytest tests/ -q -ra --cov=ml4t.data --cov-report=term-missing
 uv run mkdocs build --strict
 uv build
 ```

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/README.md

@@ -70,7 +70,7 @@ uv run pre-commit install
 uv run ruff check src/ tests/
 uv run ruff format --check src/ tests/
 uv run ty check
-uv run pytest tests/ -q
+uv run pytest tests/ -q -ra --cov=ml4t.data --cov-report=term-missing
 uv run mkdocs build --strict
 uv build
 ```

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/pyproject.toml

@@ -8,6 +8,9 @@ build-backend = "hatchling.build"
 [tool.hatch.version]
 source = "vcs"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.build.hooks.vcs]
 version-file = "src/ml4t/data/_version.py"
 
@@ -74,12 +77,14 @@ requires-python = ">=3.12,<3.15"
 
 # Core dependencies
 dependencies = [
+    "ml4t-specs>=0.2.0",
     # Data processing
-    "polars>=0.20.0",
+    "polars>=1.36.1",
     "pandas>=2.0.0",
     "numpy>=1.24.0",
     # HTTP and networking
     "httpx>=0.25.0",
+    "certifi>=2026.10.19",
     "tenacity>=8.0.0",
     "pybreaker>=1.0.0", # Circuit breaker pattern
     # Configuration and CLI
@@ -95,14 +100,14 @@ dependencies = [
     "rich>=13.0.0",
     "aiofiles>=23.0.0", # Async file I/O for async storage backends
     # Exchange calendars for session-aware operations
-    "pandas-market-calendars>=4.3.0",
+    "pandas-market-calendars>=5.4.0",
     "openpyxl>=3.1.5",
 ]
 
 [project.optional-dependencies]
 # Provider-specific dependencies
 yahoo = [
-    "yfinance>=0.2.0",
+    "yfinance>=1.5.2",
 ]
 databento = [
     "databento>=0.38.0",
@@ -181,7 +186,7 @@ dev = [
     "twine>=6.0.0",
     "pip-audit>=2.10.1",
     "databento>=0.38.0",
-    "yfinance>=0.2.0",
+    "yfinance>=1.5.2",
 ]
 
 [tool.uv]
@@ -301,6 +306,8 @@ omit = [
 ]
 
 [tool.coverage.report]
+fail_under = 85
+show_missing = true
 exclude_lines = [
     "pragma: no cover",
     "def __repr__",

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/src/ml4t/data/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)
 
 __commit_id__ = commit_id = None

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/src/ml4t/data/adjustments/__init__.py

@@ -15,9 +15,15 @@ from .core import (
     apply_dividends,
     apply_splits,
 )
+from .history import AdjustedHistory, prepare_adjusted_history
+from .inputs import apply_adjustment_factors, prepare_action_inputs
 
 __all__ = [
     "apply_splits",
     "apply_dividends",
     "apply_corporate_actions",
+    "apply_adjustment_factors",
+    "prepare_action_inputs",
+    "AdjustedHistory",
+    "prepare_adjusted_history",
 ]

ml4t_data-0.3.0/src/ml4t/data/adjustments/_history_factors.py

@@ -0,0 +1,110 @@
+"""Select supplied factor curves without recomputing their financial definition."""
+
+from datetime import datetime
+from typing import Any
+
+import pandas as pd
+import polars as pl
+
+from ml4t.data.core.revisions import select_revisions, session_label_at, utc_column, utc_cutoff
+from ml4t.specs import FeedSpec
+
+
+def prepare_supplied_factors(
+    factors: pl.DataFrame,
+    prices: pl.DataFrame,
+    actions: pl.DataFrame,
+    feed: FeedSpec,
+    as_of: datetime | str | None,
+    clock_col: str,
+) -> tuple[pl.DataFrame, dict[str, Any]]:
+    asset = FeedSpec._coerce_entity_col(feed.entity_col) or "asset"
+    keys = [asset, clock_col]
+    if "price_adjustment_factor" not in factors.columns:
+        raise ValueError("A supplied history requires price_adjustment_factor")
+    candidates = factors.join(prices.select(keys).unique(), on=keys, how="semi")
+    candidates = utc_column(candidates, "available_at", feed.timezone)
+    versioned = {"source", "revision_id"}.issubset(candidates.columns) and all(
+        not candidates[name].null_count() for name in ("source", "revision_id")
+    )
+    selected = (
+        select_revisions(candidates, [*keys, "source"], as_of)
+        if as_of is not None or versioned
+        else candidates
+    )
+    if selected.select(keys).is_duplicated().any():
+        raise ValueError("Choose one supplied factor revision per observation")
+    if prices.select(keys).join(selected.select(keys), on=keys, how="anti").height:
+        raise ValueError("Supplied factor coverage is incomplete at the requested cutoff")
+    if "basis_at" not in selected.columns:
+        selected = selected.with_columns(
+            pl.lit(None, dtype=pl.Datetime("ns", "UTC")).alias("basis_at")
+        )
+    if selected.schema["basis_at"] != pl.Date:
+        selected = utc_column(selected, "basis_at", feed.timezone)
+    if as_of is not None:
+        if selected.schema["basis_at"] == pl.Date or selected["basis_at"].null_count():
+            raise ValueError("Known basis_at instants are required for causal supplied factors")
+        if selected.filter(pl.col("basis_at") > utc_cutoff(as_of)).height:
+            raise ValueError("Supplied basis_at cannot be later than the cutoff")
+        counts = selected.group_by(asset).agg(pl.col("basis_at").n_unique())
+        if counts.filter(pl.col("basis_at") != 1).height:
+            raise ValueError("Declare one supplied share basis per asset")
+        bases = selected.select(asset, "basis_at").unique()
+        if actions.height:
+            latest = actions.group_by(asset).agg(
+                pl.col("effective_at").max().alias("last_effective_at")
+            )
+            if actions.schema["effective_at"] == pl.Date:
+                stamps = bases["basis_at"].cast(pl.Datetime("ns", "UTC")).cast(pl.Int64)
+                labels = [
+                    session_label_at(pd.Timestamp(value, unit="ns", tz="UTC"), feed)
+                    for value in stamps
+                ]
+                bases = bases.with_columns(pl.Series("basis_session", labels, dtype=pl.Date))
+                checked = bases.join(latest, on=asset, how="left")
+                stale = checked.filter(pl.col("last_effective_at") > pl.col("basis_session"))
+            else:
+                checked = bases.join(latest, on=asset, how="left")
+                stale = checked.filter(pl.col("last_effective_at") > pl.col("basis_at"))
+            if stale.height:
+                raise ValueError("Supplied share basis predates an applicable known source action")
+    if "currency" in selected.columns and "currency" in prices.columns:
+        units = prices.select(*keys, pl.col("currency").alias("price_currency")).join(
+            selected.select(*keys, pl.col("currency").alias("factor_currency")),
+            on=keys,
+        )
+        if units.filter(
+            pl.col("price_currency").is_not_null()
+            & pl.col("factor_currency").is_not_null()
+            & (pl.col("price_currency") != pl.col("factor_currency"))
+        ).height:
+            raise ValueError("Known price and supplied factor currency units conflict")
+    numeric = [
+        name
+        for name in ("price_adjustment_factor", "volume_adjustment_factor")
+        if name in selected.columns
+    ]
+    provenance = [name for name in selected.columns if name not in {*keys, *numeric}]
+    projected = selected.select(
+        *keys,
+        *numeric,
+        *(pl.col(name).alias(f"factor_{name}") for name in provenance),
+    )
+    output = (
+        prices.select(asset, feed.timestamp_col, clock_col)
+        .join(
+            projected,
+            on=keys,
+            how="left",
+            validate="1:1",
+        )
+        .drop(clock_col)
+    )
+    return output, {
+        "factor_revision_identity": "declared" if versioned else "unknown unversioned capture",
+        "factor_basis_known": not selected["basis_at"].null_count(),
+        "factor_availability_known": not selected["available_at"].null_count(),
+        "factor_rows_outside_window": factors.height - candidates.height,
+        "volume_factor_supplied": "volume_adjustment_factor" in numeric,
+    }

{ml4t_data-0.2.0 → ml4t_data-0.3.0}/src/ml4t/data/adjustments/core.py

@@ -47,6 +47,22 @@ def _future_cumulative_product(column: str) -> pl.Expr:
     return pl.col(column).shift(-1, fill_value=1.0).reverse().cum_prod().reverse()
 
 
+def _event_adjustment_factors(
+    prices: pl.DataFrame, *, split_col: str, dividend_col: str | None
+) -> pl.DataFrame:
+    """Add canonical local event factors before composing a historical share basis."""
+    price_event_factor = (
+        pl.col("close")
+        / (pl.col(split_col).cast(pl.Float64) * (pl.col("close") + pl.col(dividend_col)))
+        if dividend_col is not None
+        else 1.0 / pl.col(split_col).cast(pl.Float64)
+    )
+    return prices.with_columns(
+        price_event_factor.alias("_price_event_factor"),
+        pl.col(split_col).cast(pl.Float64).alias("_volume_event_factor"),
+    )
+
+
 def _apply_canonical_adjustments(
     prices: pl.DataFrame,
     *,
@@ -64,16 +80,7 @@ def _apply_canonical_adjustments(
     )
     df = prices.sort("date").clone()
 
-    price_event_factor = (
-        pl.col("close")
-        / (pl.col(split_col).cast(pl.Float64) * (pl.col("close") + pl.col(dividend_col)))
-        if dividend_col is not None
-        else 1.0 / pl.col(split_col).cast(pl.Float64)
-    )
-    df = df.with_columns(
-        price_event_factor.alias("_price_event_factor"),
-        pl.col(split_col).cast(pl.Float64).alias("_volume_event_factor"),
-    ).with_columns(
+    df = _event_adjustment_factors(df, split_col=split_col, dividend_col=dividend_col).with_columns(
         _future_cumulative_product("_price_event_factor").alias("price_adjustment_factor"),
         _future_cumulative_product("_volume_event_factor").alias("volume_adjustment_factor"),
     )