xrtm-data 0.2.4__tar.gz → 0.2.6__tar.gz

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xrtm-data
-Version: 0.2.4
+Version: 0.2.6
 Summary: The Snapshot Vault for XRTM.
 Author-email: XRTM Team <moy@xrtm.org>
 License-Expression: Apache-2.0

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xrtm-data"
-version = "0.2.4"
+version = "0.2.6"
 description = "The Snapshot Vault for XRTM."
 readme = "README.md"
 requires-python = ">=3.11,<3.13"

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/src/xrtm/data/__init__.py

@@ -31,7 +31,7 @@ Example:
 """
 
 # Core interfaces
-from xrtm.data.core import DataSource
+from xrtm.data.core import DataSource, DataSourceError, SourceFetchError, SourceTemporalIntegrityError
 
 # Core schemas (public API)
 from xrtm.data.core.schemas import (
@@ -46,6 +46,9 @@ from xrtm.data.core.schemas import (
 __all__ = [
     # Interfaces
     "DataSource",
+    "DataSourceError",
+    "SourceFetchError",
+    "SourceTemporalIntegrityError",
     # Schemas
     "MetadataBase",
     "ForecastQuestion",

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/src/xrtm/data/core/__init__.py

@@ -21,6 +21,6 @@ providers must implement. The core module is domain-agnostic and MUST NOT
 import from kit/ or providers/.
 """
 
-from xrtm.data.core.interfaces import DataSource
+from xrtm.data.core.interfaces import DataSource, DataSourceError, SourceFetchError, SourceTemporalIntegrityError
 
-__all__ = ["DataSource"]
+__all__ = ["DataSource", "DataSourceError", "SourceFetchError", "SourceTemporalIntegrityError"]

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/src/xrtm/data/core/interfaces.py

@@ -23,18 +23,31 @@ data provider, regardless of the source.
 Example:
     >>> from xrtm.data.core import DataSource
     >>> class MySource(DataSource):
-    ...     async def fetch_questions(self, query=None, limit=5):
+    ...     async def fetch_questions(self, query=None, limit=5, *, snapshot_time=None):
     ...         return []
     ...     async def get_question_by_id(self, question_id):
     ...         return None
 """
 
 import abc
+from datetime import datetime
 from typing import List, Optional
 
 from xrtm.data.core.schemas.forecast import ForecastQuestion
 
 
+class DataSourceError(RuntimeError):
+    r"""Base exception for data source failures."""
+
+
+class SourceFetchError(DataSourceError):
+    r"""Raised when a provider cannot fetch or decode source data."""
+
+
+class SourceTemporalIntegrityError(DataSourceError):
+    r"""Raised when a provider cannot satisfy a requested snapshot safely."""
+
+
 class DataSource(abc.ABC):
     r"""
     Abstract interface for gathering or streaming forecasting workloads.
@@ -47,18 +60,22 @@ class DataSource(abc.ABC):
 
     Example:
         >>> class LocalSource(DataSource):
-        ...     async def fetch_questions(self, query=None, limit=5):
+        ...     async def fetch_questions(self, query=None, limit=5, *, snapshot_time=None):
         ...         return [ForecastQuestion(id="1", title="Test")]
     """
 
     @abc.abstractmethod
-    async def fetch_questions(self, query: Optional[str] = None, limit: int = 5) -> List[ForecastQuestion]:
+    async def fetch_questions(
+        self, query: Optional[str] = None, limit: int = 5, *, snapshot_time: Optional[datetime] = None
+    ) -> List[ForecastQuestion]:
         r"""
         Fetch a list of forecast questions from the data source.
 
         Args:
             query: Optional search/filter string.
             limit: Maximum number of questions to return.
+            snapshot_time: Optional end-of-history timestamp. Providers that cannot
+                satisfy historical snapshots must surface a temporal integrity error.
 
         Returns:
             List of ForecastQuestion objects matching the criteria.
@@ -66,12 +83,16 @@ class DataSource(abc.ABC):
         pass
 
     @abc.abstractmethod
-    async def get_question_by_id(self, question_id: str) -> Optional[ForecastQuestion]:
+    async def get_question_by_id(
+        self, question_id: str, *, snapshot_time: Optional[datetime] = None
+    ) -> Optional[ForecastQuestion]:
         r"""
         Retrieve a single question by its unique identifier.
 
         Args:
             question_id: The unique identifier of the question.
+            snapshot_time: Optional end-of-history timestamp. Providers that cannot
+                satisfy historical snapshots must surface a temporal integrity error.
 
         Returns:
             The ForecastQuestion if found, None otherwise.
@@ -79,4 +100,4 @@ class DataSource(abc.ABC):
         pass
 
 
-__all__ = ["DataSource"]
+__all__ = ["DataSource", "DataSourceError", "SourceFetchError", "SourceTemporalIntegrityError"]

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/src/xrtm/data/core/schemas/forecast.py

@@ -28,7 +28,7 @@ Example:
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 
-from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_validator, model_validator
 
 
 class MetadataBase(BaseModel):
@@ -66,6 +66,14 @@ class MetadataBase(BaseModel):
     source_version: Optional[str] = Field(None, description="Version of the data source")
     raw_data: Optional[Dict[str, Any]] = Field(None, description="Original unprocessed data")
 
+    @field_validator("created_at", "snapshot_time", mode="after")
+    @classmethod
+    def _normalize_temporal_fields(cls, value: datetime) -> datetime:
+        r"""Store temporal boundary fields as timezone-aware UTC datetimes."""
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
     def get(self, key: str, default: Any = None) -> Any:
         r"""Backward compatibility for dict-like access."""
         return getattr(self, key, default)

{xrtm_data-0.2.4 → xrtm_data-0.2.6}/src/xrtm/data/core/schemas/trade.py

@@ -31,10 +31,17 @@ Example:
     ... )
 """
 
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+def _as_utc(value: datetime) -> datetime:
+    r"""Normalize datetimes to timezone-aware UTC without rejecting legacy naive inputs."""
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
 
 
 class TradeEvent(BaseModel):
@@ -98,6 +105,12 @@ class TradeEvent(BaseModel):
         description="Transaction hash for verification",
     )
 
+    @field_validator("timestamp", mode="after")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime) -> datetime:
+        r"""Normalize trade timestamps to UTC to make window comparisons stable."""
+        return _as_utc(value)
+
     @property
     def yes_weight(self) -> float:
         r"""Volume-weighted contribution to Yes outcome: price × amount."""
@@ -149,6 +162,27 @@ class TradeWindow(BaseModel):
         description="Identifier for the market these trades belong to",
     )
 
+    @field_validator("start_time", "end_time", mode="after")
+    @classmethod
+    def _normalize_window_boundary(cls, value: datetime) -> datetime:
+        r"""Normalize window boundaries to UTC before enforcing leakage invariants."""
+        return _as_utc(value)
+
+    @model_validator(mode="after")
+    def _validate_temporal_bounds(self) -> "TradeWindow":
+        r"""Ensure a trade window cannot contain future or pre-window events."""
+        if self.end_time < self.start_time:
+            raise ValueError("end_time must not precede start_time")
+
+        leaked = [
+            trade.timestamp
+            for trade in self.trades
+            if trade.timestamp < self.start_time or trade.timestamp > self.end_time
+        ]
+        if leaked:
+            raise ValueError("trades must fall within [start_time, end_time]")
+        return self
+
     @property
     def total_volume(self) -> float:
         r"""Total trading volume in the window."""

xrtm_data-0.2.6/src/xrtm/data/corpora/__init__.py

@@ -0,0 +1,99 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Public entry points for XRTM corpora.
+
+The package-root API intentionally stays focused on the stable registry and the
+embedded real-binary corpus. Less-stable helpers remain available from their
+submodules and are kept here as lazy compatibility exports.
+
+Example:
+    >>> from xrtm.data.corpora import get_corpus, list_available_corpora
+    >>> corpus = get_corpus("xrtm-real-binary-v1")
+    >>> metadata_list = list_available_corpora(release_gate_only=True)
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+
+from xrtm.data.corpora.forecast_importer import FORECAST_CORPUS_ID
+from xrtm.data.corpora.real_binary import (
+    REAL_BINARY_CORPUS_ID,
+    RealBinaryCorpusSource,
+    RealBinaryQuestionRecord,
+    load_real_binary_corpus,
+    load_real_binary_questions,
+    load_real_binary_resolved_outcomes,
+    validate_real_binary_corpus,
+)
+from xrtm.data.corpora.registry import (
+    CorpusAvailability,
+    CorpusManifest,
+    CorpusMetadata,
+    CorpusRegistry,
+    CorpusSplit,
+    CorpusTier,
+    LicenseType,
+    describe_corpus,
+    get_corpus,
+    get_corpus_metadata,
+    list_available_corpora,
+    prepare_corpus,
+)
+
+__all__ = [
+    "REAL_BINARY_CORPUS_ID",
+    "FORECAST_CORPUS_ID",
+    "RealBinaryQuestionRecord",
+    "RealBinaryCorpusSource",
+    "load_real_binary_corpus",
+    "load_real_binary_questions",
+    "load_real_binary_resolved_outcomes",
+    "validate_real_binary_corpus",
+    "CorpusRegistry",
+    "CorpusAvailability",
+    "CorpusMetadata",
+    "CorpusManifest",
+    "CorpusTier",
+    "LicenseType",
+    "CorpusSplit",
+    "describe_corpus",
+    "get_corpus",
+    "get_corpus_metadata",
+    "list_available_corpora",
+    "prepare_corpus",
+]
+
+_COMPAT_EXPORTS = {
+    "FORECAST_HF_DATASET": ("xrtm.data.corpora.forecast_importer", "FORECAST_HF_DATASET"),
+    "FOReCAstImporter": ("xrtm.data.corpora.forecast_importer", "FOReCAstImporter"),
+    "CorpusImporter": ("xrtm.data.corpora.importers", "CorpusImporter"),
+    "ImportManifest": ("xrtm.data.corpora.importers", "ImportManifest"),
+    "OfflineCorpusCache": ("xrtm.data.corpora.importers", "OfflineCorpusCache"),
+    "DeterministicFixtureImporter": ("xrtm.data.corpora.importers", "DeterministicFixtureImporter"),
+    "SplitConfig": ("xrtm.data.corpora.splits", "SplitConfig"),
+    "CorpusSplitter": ("xrtm.data.corpora.splits", "CorpusSplitter"),
+    "SplitAwareCorpusSource": ("xrtm.data.corpora.splits", "SplitAwareCorpusSource"),
+}
+
+def __getattr__(name: str):
+    if name in _COMPAT_EXPORTS:
+        module_name, attr_name = _COMPAT_EXPORTS[name]
+        return getattr(import_module(module_name), attr_name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+def __dir__() -> list[str]:
+    return sorted(set(__all__) | set(_COMPAT_EXPORTS))

xrtm_data-0.2.6/src/xrtm/data/corpora/_builtin_corpora.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Concrete built-in corpus registrations.
+
+This module keeps registry bootstrap and corpus-specific cache workflows out of
+``registry.py`` so the registry can stay focused on generic manifest handling.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from pathlib import Path
+from typing import List, Optional
+
+from xrtm.data.core import DataSource
+from xrtm.data.corpora.forecast_importer import FORECAST_CORPUS_ID, FOReCAstImporter
+from xrtm.data.corpora.importers import OfflineCorpusCache
+from xrtm.data.corpora.real_binary import REAL_BINARY_CORPUS_ID, RealBinaryCorpusSource
+from xrtm.data.corpora.registry import (
+    CorpusAvailability,
+    CorpusManifest,
+    CorpusMetadata,
+    CorpusSplit,
+    CorpusTier,
+    LicenseType,
+)
+
+_FORECAST_VERSION = "1.0"
+
+def build_builtin_manifests() -> List[CorpusManifest]:
+    """Build manifests for the corpora shipped with the registry bootstrap."""
+    return [
+        _build_real_binary_manifest(),
+        _build_forecast_manifest(),
+    ]
+
+def _build_real_binary_manifest() -> CorpusManifest:
+    real_binary_metadata = CorpusMetadata(
+        corpus_id=REAL_BINARY_CORPUS_ID,
+        name="XRTM Real Binary v1",
+        tier=CorpusTier.TIER_1,
+        license_type=LicenseType.APACHE_2_0,
+        description="Minimal deterministic real-world binary question corpus for CI smoke tests",
+        version="1.0",
+        release_gate_approved=True,
+        bundled=True,
+        size_estimate=25,
+        tags=["binary", "deterministic", "embedded", "seed-corpus"],
+        provenance_url="https://github.com/xrtm/xrtm",
+        license_url="https://www.apache.org/licenses/LICENSE-2.0",
+    )
+
+    return CorpusManifest(
+        corpus_id=REAL_BINARY_CORPUS_ID,
+        metadata=real_binary_metadata,
+        loader_fn=lambda: RealBinaryCorpusSource(),
+        available_splits=[CorpusSplit.FULL],
+        default_split=CorpusSplit.FULL,
+    )
+
+def _build_forecast_manifest() -> CorpusManifest:
+    forecast_metadata = CorpusMetadata(
+        corpus_id=FORECAST_CORPUS_ID,
+        name="FOReCAst (Future Outcome Reasoning and Confidence Assessment)",
+        tier=CorpusTier.TIER_2,
+        license_type=LicenseType.MIT,
+        description="Academic benchmark dataset for probabilistic forecasting from NeurIPS 2025. "
+        "1,390 resolved questions from Metaculus. Evaluation-only until Tier 1 approval.",
+        version=_FORECAST_VERSION,
+        release_gate_approved=False,
+        bundled=False,
+        size_estimate=1390,
+        tags=["forecast", "external", "evaluation-only", "probabilistic"],
+        provenance_url="https://huggingface.co/datasets/MoyYuan/FOReCAst",
+        license_url="https://opensource.org/licenses/MIT",
+        citation="FOReCAst: Future Outcome Reasoning and Confidence Assessment. NeurIPS 2025 Datasets and Benchmarks Track.",
+        extra={
+            "tier_status": "evaluation-only",
+            "promotion_required": "explicit approval needed for Tier 1 promotion",
+            "non_commercial_clause": "pending clarification",
+        },
+    )
+
+    return CorpusManifest(
+        corpus_id=FORECAST_CORPUS_ID,
+        metadata=forecast_metadata,
+        loader_fn=lambda: _load_forecast_source(forecast_metadata.version),
+        available_splits=[CorpusSplit.FULL, CorpusSplit.TRAIN, CorpusSplit.EVAL],
+        default_split=CorpusSplit.FULL,
+        importer_module="xrtm.data.corpora.forecast_importer",
+        availability_loader=lambda cache_root: _describe_forecast_corpus(
+            forecast_metadata.version,
+            cache_root=cache_root,
+        ),
+        prepare_loader=lambda cache_root, refresh, use_hf_datasets: _prepare_forecast_corpus(
+            forecast_metadata.version,
+            cache_root=cache_root,
+            refresh=refresh,
+            use_hf_datasets=use_hf_datasets,
+        ),
+    )
+
+def _describe_forecast_corpus(
+    version: str,
+    *,
+    cache_root: Optional[Path] = None,
+) -> CorpusAvailability:
+    resolved_cache_root = _resolve_cache_root(cache_root)
+    cache = OfflineCorpusCache(resolved_cache_root)
+    manifest = cache.load_manifest(FORECAST_CORPUS_ID, version)
+    import_method = manifest.metadata.get("import_method") if manifest is not None else None
+    source_mode = "preview" if import_method in {None, "fixture"} else "external-cache"
+
+    return CorpusAvailability(
+        corpus_id=FORECAST_CORPUS_ID,
+        version=version,
+        source_mode=source_mode,
+        bundled=False,
+        already_cached=manifest is not None,
+        record_count=manifest.record_count if manifest is not None else None,
+        import_method=import_method,
+        cache_root=resolved_cache_root,
+        data_dir=cache.get_corpus_dir(FORECAST_CORPUS_ID, version),
+        manifest_path=cache.get_manifest_path(FORECAST_CORPUS_ID, version),
+    )
+
+def _prepare_forecast_corpus(
+    version: str,
+    *,
+    cache_root: Optional[Path] = None,
+    refresh: bool = False,
+    use_hf_datasets: bool = True,
+) -> CorpusAvailability:
+    resolved_cache_root = _resolve_cache_root(cache_root)
+    cache = OfflineCorpusCache(resolved_cache_root)
+    if cache.is_cached(FORECAST_CORPUS_ID, version) and not refresh:
+        return _describe_forecast_corpus(version, cache_root=resolved_cache_root)
+
+    importer = FOReCAstImporter(use_hf_datasets=use_hf_datasets)
+    data_dir = cache.get_corpus_dir(FORECAST_CORPUS_ID, version)
+    manifest = importer.import_corpus(data_dir, version=version)
+    cache.save_manifest(manifest)
+    return _describe_forecast_corpus(version, cache_root=resolved_cache_root)
+
+def _load_forecast_source(version: str) -> DataSource:
+    importer = FOReCAstImporter(use_hf_datasets=False)
+    cache = OfflineCorpusCache(_resolve_cache_root())
+    data_dir = cache.get_corpus_dir(FORECAST_CORPUS_ID, version)
+    manifest = cache.load_manifest(FORECAST_CORPUS_ID, version)
+    if manifest is None:
+        warnings.warn(
+            "FOReCAst full cache not found; using the 3-record deterministic preview. "
+            "Prepare the corpus cache first to run large-scale validation.",
+            UserWarning,
+            stacklevel=4,
+        )
+        manifest = importer.import_corpus(data_dir, version=version)
+        cache.save_manifest(manifest)
+    elif manifest.metadata.get("import_method") == "fixture":
+        warnings.warn(
+            "FOReCAst cache currently contains the deterministic preview only. "
+            "Refresh the cache with the external dataset before relying on large-scale counts.",
+            UserWarning,
+            stacklevel=4,
+        )
+    return importer.load_from_manifest(manifest, data_dir)
+
+def _resolve_cache_root(cache_root: Optional[Path] = None) -> Path:
+    if cache_root is not None:
+        return cache_root
+    return Path(os.environ.get("XRTM_CORPUS_CACHE", Path.home() / ".xrtm" / "corpus-cache"))