xrtm-data 0.2.0__tar.gz → 0.2.2__tar.gz

{xrtm_data-0.2.0 → xrtm_data-0.2.2}/PKG-INFO

@@ -1,14 +1,18 @@
 Metadata-Version: 2.4
 Name: xrtm-data
-Version: 0.2.0
+Version: 0.2.2
 Summary: The Snapshot Vault for XRTM.
 Author-email: XRTM Team <moy@xrtm.org>
-License: Apache-2.0
+License-Expression: Apache-2.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: scipy>=1.11.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pyarrow>=14.0.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"

{xrtm_data-0.2.0 → xrtm_data-0.2.2}/pyproject.toml

@@ -4,17 +4,21 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xrtm-data"
-version = "0.2.0"
+version = "0.2.2"
 description = "The Snapshot Vault for XRTM."
 readme = "README.md"
 requires-python = ">=3.11"
-license = {text = "Apache-2.0"}
+license = "Apache-2.0"
 authors = [
     {name = "XRTM Team", email = "moy@xrtm.org"}
 ]
 dependencies = [
     "pydantic>=2.0.0",
     "aiohttp>=3.9.0",
+    "scipy>=1.11.0",
+    "click>=8.0.0",
+    "rich>=13.0.0",
+    "pyarrow>=14.0.0",
 ]
 
 [project.optional-dependencies]
@@ -26,12 +30,14 @@ dev = [
     "mypy>=1.0.0",
 ]
 
+[project.scripts]
+xrtm-data = "xrtm.data.cli:main"
+
 [tool.setuptools]
 package-dir = {"" = "src"}
 packages = {find = {where = ["src"], include = ["xrtm*"], namespaces = true}}
 
 [tool.pytest.ini_options]
-pythonpath = ["."]
 testpaths = ["tests"]
 asyncio_mode = "strict"
 

xrtm_data-0.2.2/src/xrtm/data/cli/__init__.py

@@ -0,0 +1,322 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""
+CLI entry point for xrtm-data.
+
+Provides commands for:
+- Collecting trade data from Polymarket
+- Fitting Beta priors from trade history
+- Caching data to Parquet files
+
+Example:
+    $ xrtm-data collect --market-id 0x... --days 30 -o trades.parquet
+    $ xrtm-data fit-prior --input trades.parquet -o prior.json
+"""
+
+import asyncio
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Optional
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
+
+from xrtm.data.version import __version__
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version=__version__)
+def main():
+    r"""xrtm-data: Data collection and preprocessing for xRTM training."""
+    pass
+
+
+@main.command()
+@click.option("--market-id", "-m", required=True, help="Polymarket market ID (hex address)")
+@click.option("--days", "-d", default=30, help="Number of days of history to fetch")
+@click.option("--start", type=click.DateTime(), help="Start date (overrides --days)")
+@click.option("--end", type=click.DateTime(), help="End date (default: now)")
+@click.option("--output", "-o", required=True, type=click.Path(), help="Output file path (.parquet or .json)")
+@click.option("--force", "-f", is_flag=True, help="Overwrite existing file")
+def collect(
+    market_id: str,
+    days: int,
+    start: Optional[datetime],
+    end: Optional[datetime],
+    output: str,
+    force: bool,
+):
+    r"""
+    Collect trade data from Polymarket.
+
+    Fetches historical trades for a given market and saves to Parquet or JSON.
+    Supports caching — will skip if output file exists unless --force is used.
+
+    Example:
+        xrtm-data collect -m 0x1234... -d 30 -o data/trades.parquet
+    """
+    output_path = Path(output)
+
+    # Check cache
+    if output_path.exists() and not force:
+        console.print(f"[yellow]⚠ File exists:[/yellow] {output_path}")
+        console.print("  Use --force to overwrite, or specify different output.")
+        return
+
+    # Calculate time range
+    end_time = end or datetime.now(timezone.utc)
+    if start:
+        start_time = start.replace(tzinfo=timezone.utc)
+    else:
+        start_time = end_time - timedelta(days=days)
+
+    console.print(Panel(
+        f"[bold blue]Collecting Polymarket Trades[/bold blue]\n"
+        f"Market: {market_id[:16]}...\n"
+        f"Range: {start_time.date()} → {end_time.date()}",
+        title="xrtm-data",
+    ))
+
+    # Run async collection
+    async def _collect():
+        from xrtm.data.providers.subgraph import PolymarketTradeSource
+
+        source = PolymarketTradeSource()
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            console=console,
+        ) as progress:
+            task = progress.add_task("Fetching trades...", total=None)
+
+            window = await source.fetch_trade_window(
+                market_id=market_id,
+                start_time=start_time,
+                end_time=end_time,
+            )
+
+            progress.update(task, description=f"Fetched {len(window.trades)} trades")
+
+        return window
+
+    window = asyncio.run(_collect())
+
+    # Save output
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if output_path.suffix == ".parquet":
+        _save_parquet(window, output_path)
+    else:
+        _save_json(window, output_path)
+
+    console.print(f"[green]✓ Saved {len(window.trades)} trades to:[/green] {output_path}")
+
+    # Summary table
+    table = Table(title="Trade Summary")
+    table.add_column("Metric", style="cyan")
+    table.add_column("Value", style="green")
+    table.add_row("Total Trades", str(window.trade_count))
+    table.add_row("Total Volume", f"${window.total_volume:,.2f}")
+    table.add_row("VWAP", f"{window.volume_weighted_price:.4f}")
+    console.print(table)
+
+
+@main.command("fit-prior")
+@click.option("--input", "-i", "input_path", required=True, type=click.Path(exists=True), help="Input trades file")
+@click.option("--output", "-o", required=True, type=click.Path(), help="Output prior file (.json)")
+@click.option("--half-life", "-h", default=24.0, help="Half-life in hours for decay weighting")
+@click.option("--min-concentration", default=2.0, help="Minimum concentration (α+β)")
+def fit_prior(input_path: str, output: str, half_life: float, min_concentration: float):
+    r"""
+    Fit a Beta prior from trade data.
+
+    Reads trade history and fits a Beta distribution using exponential
+    decay weighting (recent trades weighted more heavily).
+
+    Example:
+        xrtm-data fit-prior -i trades.parquet -o prior.json -h 24.0
+    """
+    from xrtm.data.kit.processors import fit_beta_exponential_decay
+
+    console.print(Panel(
+        f"[bold blue]Fitting Beta Prior[/bold blue]\n"
+        f"Input: {input_path}\n"
+        f"Half-life: {half_life} hours",
+        title="xrtm-data",
+    ))
+
+    # Load trades
+    trades = _load_trades(Path(input_path))
+    console.print(f"Loaded {len(trades)} trades")
+
+    # Fit prior
+    prior = fit_beta_exponential_decay(
+        trades,
+        half_life_hours=half_life,
+        min_concentration=min_concentration,
+    )
+
+    # Save
+    output_path = Path(output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    prior_dict = prior.to_distribution_dict()
+    prior_dict["metadata"] = {
+        "source": str(input_path),
+        "trade_count": len(trades),
+        "half_life_hours": half_life,
+        "fitted_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    with open(output_path, "w") as f:
+        json.dump(prior_dict, f, indent=2)
+
+    console.print(f"[green]✓ Saved prior to:[/green] {output_path}")
+
+    # Summary
+    low, high = prior.credible_interval(0.9)
+    table = Table(title="Fitted Prior")
+    table.add_column("Parameter", style="cyan")
+    table.add_column("Value", style="green")
+    table.add_row("α (alpha)", f"{prior.alpha:.4f}")
+    table.add_row("β (beta)", f"{prior.beta:.4f}")
+    table.add_row("Mean", f"{prior.mean:.4f}")
+    table.add_row("90% CI", f"[{low:.4f}, {high:.4f}]")
+    console.print(table)
+
+
+@main.command("info")
+@click.argument("file_path", type=click.Path(exists=True))
+def info(file_path: str):
+    r"""
+    Show information about a data file.
+
+    Displays summary statistics for trade files or prior files.
+    """
+    path = Path(file_path)
+
+    if path.suffix == ".json":
+        with open(path) as f:
+            data = json.load(f)
+
+        if "family" in data:
+            # It's a prior
+            console.print(Panel(
+                f"[bold]Prior File[/bold]\n"
+                f"Family: {data['family']}\n"
+                f"α: {data.get('alpha', 'N/A')}\n"
+                f"β: {data.get('beta', 'N/A')}",
+                title=path.name,
+            ))
+        else:
+            console.print(f"JSON file with {len(data)} keys")
+    elif path.suffix == ".parquet":
+        import pyarrow.parquet as pq
+
+        table = pq.read_table(path)
+        console.print(Panel(
+            f"[bold]Parquet File[/bold]\n"
+            f"Rows: {table.num_rows}\n"
+            f"Columns: {table.column_names}",
+            title=path.name,
+        ))
+
+
+def _save_parquet(window, path: Path) -> None:
+    r"""Save TradeWindow to Parquet format."""
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    data = {
+        "price": [t.price for t in window.trades],
+        "amount": [t.amount for t in window.trades],
+        "timestamp": [t.timestamp.isoformat() for t in window.trades],
+        "maker": [t.maker for t in window.trades],
+        "taker": [t.taker for t in window.trades],
+    }
+
+    table = pa.table(data)
+    pq.write_table(table, path)
+
+
+def _save_json(window, path: Path) -> None:
+    r"""Save TradeWindow to JSON format."""
+    data = {
+        "market_id": window.market_id,
+        "start_time": window.start_time.isoformat(),
+        "end_time": window.end_time.isoformat(),
+        "trades": [
+            {
+                "price": t.price,
+                "amount": t.amount,
+                "timestamp": t.timestamp.isoformat(),
+                "maker": t.maker,
+                "taker": t.taker,
+            }
+            for t in window.trades
+        ],
+    }
+
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def _load_trades(path: Path) -> list:
+    r"""Load trades from Parquet or JSON."""
+    from xrtm.data.core.schemas import TradeEvent
+
+    if path.suffix == ".parquet":
+        import pyarrow.parquet as pq
+
+        table = pq.read_table(path)
+        return [
+            TradeEvent(
+                price=row["price"],
+                amount=row["amount"],
+                timestamp=datetime.fromisoformat(row["timestamp"]),
+                maker=row["maker"],
+                taker=row["taker"],
+            )
+            for row in table.to_pylist()
+        ]
+    else:
+        with open(path) as f:
+            data = json.load(f)
+
+        return [
+            TradeEvent(
+                price=t["price"],
+                amount=t["amount"],
+                timestamp=datetime.fromisoformat(t["timestamp"]),
+                maker=t["maker"],
+                taker=t["taker"],
+            )
+            for t in data.get("trades", data)
+        ]
+
+
+if __name__ == "__main__":
+    main()
+
+
+__all__ = ["main"]

{xrtm_data-0.2.0 → xrtm_data-0.2.2}/src/xrtm/data/core/schemas/__init__.py

@@ -14,10 +14,9 @@
 # limitations under the License.
 
 r"""
-Core data schemas for xrtm-data.
+Core schemas for xrtm-data.
 
-This module exports all foundational Pydantic models used across the
-xrtm ecosystem for representing forecasting data structures.
+This module exports all Pydantic schemas used across the xrtm ecosystem.
 """
 
 from xrtm.data.core.schemas.forecast import (
@@ -28,12 +27,21 @@ from xrtm.data.core.schemas.forecast import (
     ForecastQuestion,
     MetadataBase,
 )
+from xrtm.data.core.schemas.prior import BetaPrior, PriorState
+from xrtm.data.core.schemas.trade import TradeEvent, TradeWindow
 
 __all__ = [
+    # Forecast schemas
     "MetadataBase",
     "ForecastQuestion",
     "ForecastOutput",
     "CausalNode",
     "CausalEdge",
     "ConfidenceInterval",
+    # Prior schemas
+    "BetaPrior",
+    "PriorState",
+    # Trade schemas
+    "TradeEvent",
+    "TradeWindow",
 ]

{xrtm_data-0.2.0 → xrtm_data-0.2.2}/src/xrtm/data/core/schemas/forecast.py

@@ -28,7 +28,7 @@ Example:
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 
-from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
 
 
 class MetadataBase(BaseModel):
@@ -202,6 +202,41 @@ class ForecastOutput(BaseModel):
     calibration_metrics: Dict[str, Any] = Field(default_factory=dict, description="Performance metrics")
     metadata: MetadataBase = Field(default_factory=MetadataBase)  # type: ignore[arg-type]
 
+    @model_validator(mode="before")
+    @classmethod
+    def _apply_reasoning_trace_alias(cls, data: Any) -> Any:
+        r"""Accept governance ``reasoning_trace`` as an alias for runtime trace fields."""
+        if not isinstance(data, dict) or "reasoning_trace" not in data:
+            return data
+
+        trace = data["reasoning_trace"]
+        updated = dict(data)
+        if isinstance(trace, dict):
+            if "reasoning" not in updated and isinstance(trace.get("narrative"), str):
+                updated["reasoning"] = trace["narrative"]
+
+            causal_graph = trace.get("causal_graph")
+            if isinstance(causal_graph, dict):
+                if "logical_trace" not in updated and "nodes" in causal_graph:
+                    updated["logical_trace"] = causal_graph["nodes"]
+                if "logical_edges" not in updated and "edges" in causal_graph:
+                    updated["logical_edges"] = causal_graph["edges"]
+        elif isinstance(trace, list) and "logical_trace" not in updated:
+            updated["logical_trace"] = trace
+
+        return updated
+
+    @property
+    def reasoning_trace(self) -> Dict[str, Any]:
+        r"""Governance-compatible alias for the narrative and causal graph trace."""
+        return {
+            "narrative": self.reasoning,
+            "causal_graph": {
+                "nodes": [node.model_dump(exclude_none=True) for node in self.logical_trace],
+                "edges": [edge.model_dump(exclude_none=True) for edge in self.logical_edges],
+            },
+        }
+
     @property
     def confidence(self) -> float:
         r"""Backward compatibility alias for probability."""
@@ -227,15 +262,17 @@ class ForecastOutput(BaseModel):
         except ImportError:
             raise ImportError("networkx is required for to_networkx(). Install it with 'uv add networkx'.")
         dg = nx.DiGraph()
-        for node in self.logical_trace:
-            dg.add_node(
+        dg.add_nodes_from(
+            (
                 node.node_id,
-                event=node.event,
-                probability=node.probability,
-                description=node.description,
+                {"event": node.event, "probability": node.probability, "description": node.description},
             )
-        for edge in self.logical_edges:
-            dg.add_edge(edge.source, edge.target, weight=edge.weight, description=edge.description)
+            for node in self.logical_trace
+        )
+        dg.add_edges_from(
+            (edge.source, edge.target, {"weight": edge.weight, "description": edge.description})
+            for edge in self.logical_edges
+        )
         return dg