ml4t-data 0.2.0__tar.gz → 0.2.1__tar.gz

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ml4t-data
-Version: 0.2.0
+Version: 0.2.1
 Summary: Market data acquisition, storage, and update workflows for machine learning for trading.
 Project-URL: Homepage, https://www.ml4trading.io/
 Project-URL: Documentation, https://www.ml4trading.io/docs/data/
@@ -179,7 +179,7 @@ uv run pre-commit install
 uv run ruff check src/ tests/
 uv run ruff format --check src/ tests/
 uv run ty check
-uv run pytest tests/ -q
+uv run pytest tests/ -q -ra --cov=ml4t.data --cov-report=term-missing
 uv run mkdocs build --strict
 uv build
 ```

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/README.md

@@ -70,7 +70,7 @@ uv run pre-commit install
 uv run ruff check src/ tests/
 uv run ruff format --check src/ tests/
 uv run ty check
-uv run pytest tests/ -q
+uv run pytest tests/ -q -ra --cov=ml4t.data --cov-report=term-missing
 uv run mkdocs build --strict
 uv build
 ```

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/pyproject.toml

@@ -301,6 +301,8 @@ omit = [
 ]
 
 [tool.coverage.report]
+fail_under = 85
+show_missing = true
 exclude_lines = [
     "pragma: no cover",
     "def __repr__",

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.2.1'
+__version_tuple__ = version_tuple = (0, 2, 1)
 
 __commit_id__ = commit_id = None

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/config/models.py

@@ -55,6 +55,8 @@ class ProviderType(StrEnum):
     FAMA_FRENCH = "fama_french"
     KALSHI = "kalshi"
     POLYMARKET = "polymarket"
+    POLYMARKET_US = "polymarket_us"
+    FORECASTEX = "forecastex"
     COINGECKO = "coingecko"
     BINANCE = "binance"
     BINANCE_PUBLIC = "binance_public"

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/macro/downloader.py

@@ -30,6 +30,7 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass, field
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any
 
@@ -269,10 +270,18 @@ class MacroDataManager:
 
             ticker = yf_mapping[series_id]
             try:
+                # yfinance treats end as exclusive. Config dates are inclusive.
+                end = self.config.end
+                try:
+                    end = (datetime.strptime(end, "%Y-%m-%d") + timedelta(days=1)).strftime(
+                        "%Y-%m-%d"
+                    )
+                except ValueError:
+                    pass
                 df = yf.download(
                     ticker,
                     start=self.config.start,
-                    end=self.config.end,
+                    end=end,
                     progress=False,
                     auto_adjust=True,
                     threads=False,

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/providers/AGENTS.md

@@ -12,6 +12,7 @@ service limitations are documented in `docs/providers/`.
 | `protocols.py` | Structural contracts used by callers and type checking |
 | `registry.py` | Provider discovery, capabilities, aliases, and factory metadata |
 | `mixins/` | Shared HTTP-session and rate-limit behavior |
+| `prediction_markets.py` | Shared resolution-outcome vocabulary and trade schema for event-contract adapters |
 | `synthetic.py`, `mock.py` | Deterministic offline and testing providers |
 | Other top-level modules | One external or packaged data source per adapter |
 

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/providers/__init__.py

@@ -74,6 +74,16 @@ try:
 except ImportError:
     PolymarketProvider = None  # type: ignore
 
+try:
+    from ml4t.data.providers.polymarket_us import PolymarketUSProvider
+except ImportError:
+    PolymarketUSProvider = None  # type: ignore
+
+try:
+    from ml4t.data.providers.forecastex import ForecastExProvider
+except ImportError:
+    ForecastExProvider = None  # type: ignore
+
 # Crypto providers
 from ml4t.data.providers.coingecko import CoinGeckoProvider
 
@@ -151,6 +161,8 @@ __all__ = [
     # Prediction market providers
     "KalshiProvider",
     "PolymarketProvider",
+    "PolymarketUSProvider",
+    "ForecastExProvider",
     # Crypto providers
     "CoinGeckoProvider",
     "BinanceProvider",

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/providers/base.py

@@ -31,10 +31,12 @@ Example (composing mixins):
 
 from __future__ import annotations
 
+import math
 import os
 from abc import ABC, abstractmethod
 from typing import Any, ClassVar
 
+import httpx
 import polars as pl
 import structlog
 from tenacity import (
@@ -45,7 +47,12 @@ from tenacity import (
     wait_exponential,
 )
 
-from ml4t.data.core.exceptions import NetworkError
+from ml4t.data.core.exceptions import (
+    DataValidationError,
+    NetworkError,
+    RateLimitError,
+    SymbolNotFoundError,
+)
 from ml4t.data.providers.mixins.circuit_breaker import CircuitBreaker, CircuitBreakerMixin
 from ml4t.data.providers.mixins.rate_limit import RateLimitMixin
 from ml4t.data.providers.mixins.session import SessionMixin
@@ -73,6 +80,29 @@ def _provider_error_is_retryable(error: BaseException) -> bool:
     return isinstance(error, NetworkError) and error.retryable
 
 
+# Shared retry policy: three attempts, exponential backoff, provider ``retry_after`` honored.
+provider_retry = retry(
+    stop=stop_after_attempt(3),
+    wait=_provider_retry_wait,
+    retry=retry_if_exception(_provider_error_is_retryable),
+    reraise=True,
+)
+
+
+def retry_after_seconds(response: httpx.Response) -> float | None:
+    """Return a non-negative ``Retry-After`` delay in seconds, or None when absent or invalid."""
+    value = response.headers.get("Retry-After")
+    if value is None:
+        return None
+    try:
+        delay = float(value)
+    except ValueError:
+        return None
+    if not math.isfinite(delay):
+        return None
+    return max(delay, 0.0)
+
+
 # Re-export for backward compatibility
 __all__ = [
     "BaseProvider",
@@ -81,6 +111,8 @@ __all__ = [
     "circuit_breaker",
     "OHLCVProvider",
     "ProviderCapabilities",
+    "provider_retry",
+    "retry_after_seconds",
 ]
 
 
@@ -196,12 +228,7 @@ class BaseProvider(
             rate_limit=self.DEFAULT_RATE_LIMIT,
         )
 
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=_provider_retry_wait,
-        retry=retry_if_exception(_provider_error_is_retryable),
-        reraise=True,
-    )
+    @provider_retry
     def fetch_ohlcv(
         self,
         symbol: str,
@@ -271,6 +298,94 @@ class BaseProvider(
 
         return validated_data
 
+    def _classify_error_response(self, response: httpx.Response) -> Exception | None:
+        """Return a provider-specific exception for an error response, or None for the default.
+
+        Override to mark a service-specific failure as transient (for example an edge-layer
+        rejection). The default classification in ``_request`` applies when this returns None.
+        """
+        return None
+
+    @provider_retry
+    def _request(
+        self,
+        url: str,
+        *,
+        resource: str,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> httpx.Response:
+        """Rate-limited GET that maps transport and HTTP failures to shared exceptions.
+
+        Transient failures (transport errors, HTTP 429 and 5xx) raise retryable errors and are
+        retried by the shared ``provider_retry`` policy, honoring ``Retry-After``.
+
+        Args:
+            url: Absolute request URL.
+            resource: Human-readable name of the requested resource, used in errors.
+            params: Query parameters.
+            headers: Request headers.
+
+        Returns:
+            The successful (2xx) response.
+
+        Raises:
+            RateLimitError: HTTP 429.
+            SymbolNotFoundError: HTTP 404; ``symbol`` is ``resource``.
+            NetworkError: Transport failure, HTTP 5xx (retryable) or other HTTP errors.
+        """
+        self._acquire_rate_limit()
+        try:
+            response = self.session.get(url, params=params, headers=headers)
+        except httpx.HTTPError as err:
+            raise NetworkError(
+                provider=self.name, message=f"Request failed for {resource}: {err}"
+            ) from err
+
+        if 200 <= response.status_code < 300:
+            return response
+        custom = self._classify_error_response(response)
+        if custom is not None:
+            raise custom
+        if response.status_code == 429:
+            raise RateLimitError(provider=self.name, retry_after=retry_after_seconds(response))
+        if response.status_code == 404:
+            raise SymbolNotFoundError(provider=self.name, symbol=resource)
+        message = f"HTTP {response.status_code} for {resource}: {response.text[:200]}"
+        raise NetworkError(
+            provider=self.name,
+            message=message,
+            retryable=response.status_code >= 500,
+        )
+
+    def _request_json(
+        self,
+        url: str,
+        *,
+        resource: str,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """Run ``_request`` and parse a JSON object body.
+
+        Raises:
+            DataValidationError: The body is not valid JSON or not a JSON object.
+        """
+        response = self._request(url, resource=resource, params=params, headers=headers)
+        try:
+            payload = response.json()
+        except ValueError as err:
+            raise DataValidationError(
+                provider=self.name,
+                message=f"Malformed JSON response for {resource}",
+            ) from err
+        if not isinstance(payload, dict):
+            raise DataValidationError(
+                provider=self.name,
+                message=f"Expected a JSON object for {resource}, got {type(payload).__name__}",
+            )
+        return payload
+
     def _create_empty_dataframe(self) -> pl.DataFrame:
         """Create an empty DataFrame with canonical OHLCV schema.
 

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/providers/fama_french.py

@@ -41,6 +41,7 @@ from __future__ import annotations
 
 import io
 import zipfile
+from calendar import monthrange
 from datetime import datetime
 from pathlib import Path
 from typing import Any, ClassVar, Literal
@@ -730,11 +731,11 @@ class FamaFrenchProvider(BaseProvider):
             )
             df = df.filter(pl.col("timestamp") >= start_date)
         if end:
-            end_date = (
-                datetime.strptime(end[:10], "%Y-%m-%d").date()
-                if len(end) >= 10
-                else datetime.strptime(end + "-28", "%Y-%m-%d").date()
-            )
+            if len(end) >= 10:
+                end_date = datetime.strptime(end[:10], "%Y-%m-%d").date()
+            else:
+                year, month = (int(part) for part in end.split("-", 1))
+                end_date = datetime(year, month, monthrange(year, month)[1]).date()
             df = df.filter(pl.col("timestamp") <= end_date)
 
         self.logger.info(

{ml4t_data-0.2.0 → ml4t_data-0.2.1}/src/ml4t/data/providers/finnhub.py

@@ -4,7 +4,7 @@ Endpoint availability, request quotas, and historical depth depend on the accoun
 """
 
 import os
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any, ClassVar
 
 import polars as pl
@@ -34,6 +34,19 @@ from ml4t.data.providers.fundamentals import (
 logger = structlog.get_logger()
 
 
+def _utc_unix_timestamp(value: str, *, inclusive_date: bool = False) -> int:
+    """Parse a date or datetime as a UTC unix timestamp.
+
+    A date-only end bound covers that UTC calendar day. An explicit datetime
+    is used as given, with naive values treated as UTC.
+    """
+    parsed = datetime.fromisoformat(value)
+    parsed = parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed.astimezone(UTC)
+    if inclusive_date and "T" not in value and len(value) == 10:
+        parsed = parsed.replace(hour=23, minute=59, second=59)
+    return int(parsed.timestamp())
+
+
 class FinnhubProvider(BaseProvider):
     """Fetch Finnhub market data through synchronous provider methods.
 
@@ -196,12 +209,11 @@ class FinnhubProvider(BaseProvider):
                 value=frequency,
             )
 
-        # Convert dates to unix timestamps
+        # Convert dates to unix timestamps. Date-only bounds are UTC calendar
+        # days, with the end date inclusive through 23:59:59.
         try:
-            start_dt = datetime.fromisoformat(start)
-            end_dt = datetime.fromisoformat(end)
-            start_ts = int(start_dt.timestamp())
-            end_ts = int(end_dt.timestamp())
+            start_ts = _utc_unix_timestamp(start)
+            end_ts = _utc_unix_timestamp(end, inclusive_date=True)
         except ValueError as err:
             raise DataValidationError(
                 provider="finnhub",