probalytics 0.1.0__tar.gz → 0.2.0__tar.gz

{probalytics-0.1.0 → probalytics-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: probalytics
-Version: 0.1.0
+Version: 0.2.0
 Summary: Python client for Probalytics market, fill, and orderbook data
 Project-URL: Homepage, https://github.com/Probalytics/probalytics-python
 Project-URL: Repository, https://github.com/Probalytics/probalytics-python
@@ -218,7 +218,13 @@ book = market.orderbook_snapshots(
 )
 ```
 
-Snapshots include market identifiers, outcome, bids, asks, and timestamp.
+Each snapshot is a complete replacement book for one outcome. Rows include
+`indexed_at`, `hash`, `state`, `continuity`, and `path_index` in addition to the
+market identifiers, outcome, bids, asks, and source timestamp.
+
+Use `state="VERIFIED"` when you only need books reconstructed without gaps. Use
+`continuity` separately to distinguish rows connected to the preceding
+published state from fresh starting points.
 
 ## Choose Polars or pandas
 
@@ -303,6 +309,8 @@ directly into SQL strings.
 - `market`
 - `market_id` or list of market IDs
 - `market_platform_id` or list of market platform IDs
+- `state` or list of states
+- `continuity` or list of continuity values
 - `limit`
 
 ## Local Development

{probalytics-0.1.0 → probalytics-0.2.0}/README.md

@@ -186,7 +186,13 @@ book = market.orderbook_snapshots(
 )
 ```
 
-Snapshots include market identifiers, outcome, bids, asks, and timestamp.
+Each snapshot is a complete replacement book for one outcome. Rows include
+`indexed_at`, `hash`, `state`, `continuity`, and `path_index` in addition to the
+market identifiers, outcome, bids, asks, and source timestamp.
+
+Use `state="VERIFIED"` when you only need books reconstructed without gaps. Use
+`continuity` separately to distinguish rows connected to the preceding
+published state from fresh starting points.
 
 ## Choose Polars or pandas
 
@@ -271,6 +277,8 @@ directly into SQL strings.
 - `market`
 - `market_id` or list of market IDs
 - `market_platform_id` or list of market platform IDs
+- `state` or list of states
+- `continuity` or list of continuity values
 - `limit`
 
 ## Local Development

{probalytics-0.1.0 → probalytics-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "probalytics"
-version = "0.1.0"
+version = "0.2.0"
 description = "Python client for Probalytics market, fill, and orderbook data"
 readme = "README.md"
 requires-python = ">=3.11"

{probalytics-0.1.0 → probalytics-0.2.0}/src/probalytics/__init__.py

@@ -7,7 +7,9 @@ from probalytics.models import (
     MarketStatus,
     MarketType,
     OrderSide,
+    OrderbookContinuity,
     OrderbookSnapshot,
+    OrderbookState,
     Outcome,
     Platform,
     Resolution,
@@ -21,7 +23,9 @@ __all__ = [
     "MarketStatus",
     "MarketType",
     "OrderSide",
+    "OrderbookContinuity",
     "OrderbookSnapshot",
+    "OrderbookState",
     "Outcome",
     "ClickHouseClient",
     "Platform",

{probalytics-0.1.0 → probalytics-0.2.0}/src/probalytics/clickhouse.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from collections.abc import Sequence
 from datetime import datetime
 from enum import Enum
@@ -156,7 +157,9 @@ class ClickHouseClient:
             SELECT
                 id, market_id, market_platform_id, platform, platform_id, outcome,
                 size, price, normalized_price, taker_side, taker_cash_flow,
-                maker_cash_flow, taker_id, maker_id, fee, timestamp
+                maker_cash_flow, taker_id, maker_id, fee,
+                source_block_number, source_tx_hash, source_log_index,
+                timestamp, indexed_at, toJSONString(metadata) AS metadata
             FROM fills
             {where}
             ORDER BY timestamp ASC, id ASC
@@ -176,6 +179,8 @@ class ClickHouseClient:
         market: Market | str | UUID | None = None,
         market_id: IDFilter = None,
         market_platform_id: StringFilter = None,
+        state: StringFilter = None,
+        continuity: StringFilter = None,
         limit: int = 1000,
         frame: FrameKind = "polars",
     ) -> Any:
@@ -188,6 +193,8 @@ class ClickHouseClient:
                 _filter("platform", "platform", platform),
                 _filter("market_id", "market_id", market_id),
                 _filter("market_platform_id", "market_platform_id", market_platform_id),
+                _filter("state", "state", state),
+                _filter("continuity", "continuity", continuity),
             ]
         )
         params["limit"] = limit
@@ -195,10 +202,10 @@ class ClickHouseClient:
             SELECT
                 market_id, market_platform_id, platform,
                 {OUTCOME_FRAME_EXPR} AS outcome,
-                bids, asks, timestamp
+                bids, asks, timestamp, indexed_at, hash, state, continuity, path_index
             FROM orderbook_snapshots
             {where}
-            ORDER BY timestamp ASC
+            ORDER BY timestamp ASC, path_index ASC, indexed_at ASC
             LIMIT %(limit)s
         """
         return self._query_frame(query, params, frame)
@@ -285,7 +292,9 @@ class ClickHouseClient:
                 id, market_id, market_platform_id, platform, platform_id,
                 {OUTCOME_FRAME_EXPR} AS outcome,
                 size, price, normalized_price, taker_side, taker_cash_flow,
-                maker_cash_flow, taker_id, maker_id, fee, timestamp
+                maker_cash_flow, taker_id, maker_id, fee,
+                source_block_number, source_tx_hash, source_log_index,
+                timestamp, indexed_at, toJSONString(metadata) AS metadata
             FROM fills
             {where}
             ORDER BY timestamp ASC, id ASC
@@ -312,6 +321,8 @@ class ClickHouseClient:
             columnar=True,
         )
         data = {column[0]: values for column, values in zip(columns, columns_data, strict=True)}
+        if "metadata" in data:
+            data["metadata"] = [_json_object(value) for value in data["metadata"]]
         return dataframe_to_frame(data, frame)
 
 
@@ -378,6 +389,8 @@ def _market_row(row: dict[str, Any]) -> dict[str, Any]:
             "outcome_payouts": [_outcome_payout(value) for value in row.get("resolution_outcome_payouts") or []],
             "resolved_by": row.get("resolution_resolved_by") or "",
             "resolved_at": row.get("resolution_resolved_at"),
+            "source_block_number": row.get("resolution_source_block_number"),
+            "source_tx_hash": row.get("resolution_source_tx_hash"),
         }
     return {
         "id": row["id"],
@@ -389,7 +402,7 @@ def _market_row(row: dict[str, Any]) -> dict[str, Any]:
         "description": row.get("description", ""),
         "category": row.get("category", ""),
         "tags": row.get("tags", []),
-        "market_type": row.get("market_type", "UNKNOWN"),
+        "market_type": row["market_type"],
         "outcomes": [_outcome(value) for value in row.get("outcomes", [])],
         "status": row["status"],
         "created_at": row["created_at"],
@@ -399,6 +412,9 @@ def _market_row(row: dict[str, Any]) -> dict[str, Any]:
         "end_date": row.get("end_date"),
         "reset_at": row.get("reset_at"),
         "resolution": resolution,
+        "source_block_number": row.get("source_block_number", 0),
+        "source_tx_hash": row.get("source_tx_hash", ""),
+        "indexed_at": row["indexed_at"],
     }
 
 
@@ -406,9 +422,19 @@ def _fill_row(row: dict[str, Any]) -> dict[str, Any]:
     return {
         **row,
         "outcome": _outcome(row["outcome"]),
+        "metadata": _json_object(row.get("metadata")),
     }
 
 
+def _json_object(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError:
+        return value
+
+
 def _outcome(value: Any) -> dict[str, Any]:
     if isinstance(value, dict):
         return {

{probalytics-0.1.0 → probalytics-0.2.0}/src/probalytics/client.py

@@ -174,6 +174,8 @@ class ProbalyticsClient:
         market: Market | str | UUID | None = None,
         market_id: IDFilter = None,
         market_platform_id: StringFilter = None,
+        state: StringFilter = None,
+        continuity: StringFilter = None,
         frame: FrameKind | None = None,
         **filters: Any,
     ) -> Any:
@@ -184,6 +186,8 @@ class ProbalyticsClient:
             market=market,
             market_id=market_id,
             market_platform_id=market_platform_id,
+            state=state,
+            continuity=continuity,
             frame=validate_frame(frame or self.frame),
             **filters,
         )

{probalytics-0.1.0 → probalytics-0.2.0}/src/probalytics/models.py

@@ -14,8 +14,6 @@ from probalytics._frames import FrameKind
 class Platform(StrEnum):
     POLYMARKET = "POLYMARKET"
     KALSHI = "KALSHI"
-    PREDICTIT = "PREDICTIT"
-    UNKNOWN = "UNKNOWN"
 
 
 class MarketStatus(StrEnum):
@@ -32,7 +30,6 @@ class MarketType(StrEnum):
     SCALAR = "SCALAR"
     PARLAY = "PARLAY"
     PERPETUAL = "PERPETUAL"
-    UNKNOWN = "UNKNOWN"
 
 
 class ResolutionType(StrEnum):
@@ -46,6 +43,16 @@ class OrderSide(StrEnum):
     SELL = "SELL"
 
 
+class OrderbookState(StrEnum):
+    VERIFIED = "VERIFIED"
+    INTERMEDIATE = "INTERMEDIATE"
+
+
+class OrderbookContinuity(StrEnum):
+    CONTIGUOUS = "CONTIGUOUS"
+    RESET = "RESET"
+
+
 class ProbalyticsModel(BaseModel):
     model_config = ConfigDict(use_enum_values=True)
 
@@ -68,6 +75,8 @@ class Resolution(ProbalyticsModel):
     outcome_payouts: list[OutcomePayout] = Field(default_factory=list)
     resolved_by: str = ""
     resolved_at: datetime | None = None
+    source_block_number: int | None = None
+    source_tx_hash: str | None = None
 
 
 class Market(ProbalyticsModel):
@@ -82,7 +91,7 @@ class Market(ProbalyticsModel):
     description: str = ""
     category: str = ""
     tags: list[str] = Field(default_factory=list)
-    market_type: MarketType = MarketType.UNKNOWN
+    market_type: MarketType
     outcomes: list[Outcome] = Field(default_factory=list)
     status: MarketStatus
     created_at: datetime
@@ -92,6 +101,9 @@ class Market(ProbalyticsModel):
     end_date: datetime | None = None
     reset_at: datetime | None = None
     resolution: Resolution | None = None
+    source_block_number: int = 0
+    source_tx_hash: str = ""
+    indexed_at: datetime
 
     def fills(
         self,
@@ -143,6 +155,8 @@ class Market(ProbalyticsModel):
         start_time: datetime | str,
         end_time: datetime | str,
         client: Any | None = None,
+        state: str | list[str] | None = None,
+        continuity: str | list[str] | None = None,
         limit: int = 1000,
         frame: FrameKind | None = None,
     ) -> Any:
@@ -150,6 +164,8 @@ class Market(ProbalyticsModel):
             market=self,
             start_time=start_time,
             end_time=end_time,
+            state=state,
+            continuity=continuity,
             limit=limit,
             frame=frame,
         )
@@ -183,7 +199,12 @@ class Fill(ProbalyticsModel):
     taker_id: str | None = None
     maker_id: str | None = None
     fee: Decimal
+    source_block_number: int | None = None
+    source_tx_hash: str | None = None
+    source_log_index: int | None = None
     timestamp: datetime
+    indexed_at: datetime
+    metadata: dict[str, Any] = Field(default_factory=dict)
 
 
 class BookLevel(ProbalyticsModel):
@@ -199,6 +220,11 @@ class OrderbookSnapshot(ProbalyticsModel):
     bids: list[BookLevel] = Field(default_factory=list)
     asks: list[BookLevel] = Field(default_factory=list)
     timestamp: datetime
+    indexed_at: datetime
+    hash: int
+    state: OrderbookState
+    continuity: OrderbookContinuity
+    path_index: int
 
 
 def model_records(models: list[ProbalyticsModel]) -> list[dict[str, Any]]:

{probalytics-0.1.0 → probalytics-0.2.0}/tests/sample_clickhouse_data.py

@@ -82,6 +82,10 @@ ORDERBOOK_SNAPSHOT_COLUMNS = (
     "asks",
     "timestamp",
     "indexed_at",
+    "hash",
+    "state",
+    "continuity",
+    "path_index",
 )
 
 MARKET_ROWS = [
@@ -159,7 +163,7 @@ FILL_ROWS = [
         Decimal("0.45"),
         Decimal("0.45"),
         "BUY",
-        Decimal("-4.5"),
+        Decimal("4.5"),
         Decimal("4.5"),
         "trader-a",
         "maker-a",
@@ -182,7 +186,7 @@ FILL_ROWS = [
         Decimal("0.60"),
         "SELL",
         Decimal("3"),
-        Decimal("-3"),
+        Decimal("3"),
         "trader-b",
         "maker-b",
         Decimal("0"),
@@ -204,6 +208,10 @@ ORDERBOOK_SNAPSHOT_ROWS = [
         [(Decimal("0.46"), Decimal("8"))],
         SAMPLE_TIME.replace(second=1),
         SAMPLE_TIME.replace(second=1),
+        101,
+        "VERIFIED",
+        "RESET",
+        0,
     ),
     (
         MARKET_B,
@@ -214,6 +222,10 @@ ORDERBOOK_SNAPSHOT_ROWS = [
         [(Decimal("0.61"), Decimal("6"))],
         SAMPLE_TIME.replace(second=2),
         SAMPLE_TIME.replace(second=2),
+        202,
+        "VERIFIED",
+        "CONTIGUOUS",
+        0,
     ),
 ]
 
@@ -237,7 +249,7 @@ def insert_rows(client: DriverClient, table: str, columns: tuple[str, ...], rows
 CREATE_MARKETS = """
 CREATE TABLE markets (
     id UUID,
-    platform Enum('POLYMARKET', 'KALSHI', 'PREDICTIT', 'UNKNOWN'),
+    platform Enum('POLYMARKET', 'KALSHI'),
     platform_id String,
     slug String,
     url String,
@@ -245,7 +257,7 @@ CREATE TABLE markets (
     description String,
     category LowCardinality(String),
     tags Array(LowCardinality(String)),
-    market_type Enum('UNKNOWN', 'BINARY', 'MULTIPLE', 'SCALAR', 'PARLAY', 'PERPETUAL'),
+    market_type Enum('BINARY', 'MULTIPLE', 'SCALAR', 'PARLAY', 'PERPETUAL'),
     outcomes Array(Tuple(id UUID, platform_id String, name String, index UInt8)),
     created_at DateTime64(3),
     opened_at Nullable(DateTime64(3)),
@@ -273,7 +285,7 @@ CREATE TABLE fills (
     id UUID,
     market_id UUID,
     market_platform_id String,
-    platform Enum('POLYMARKET', 'KALSHI', 'PREDICTIT', 'UNKNOWN'),
+    platform Enum('POLYMARKET', 'KALSHI'),
     platform_id String,
     outcome Tuple(id UUID, platform_id String, name String, index UInt8),
     size Decimal128(18),
@@ -289,7 +301,8 @@ CREATE TABLE fills (
     source_tx_hash Nullable(String),
     source_log_index Nullable(UInt32),
     timestamp DateTime64(6),
-    indexed_at DateTime64(3)
+    indexed_at DateTime64(9),
+    metadata JSON DEFAULT '{}'
 ) ENGINE = ReplacingMergeTree(indexed_at)
 ORDER BY (platform, market_id, timestamp, id)
 PARTITION BY toYYYYMM(timestamp)
@@ -299,13 +312,17 @@ CREATE_ORDERBOOK_SNAPSHOTS = """
 CREATE TABLE orderbook_snapshots (
     market_id UUID,
     market_platform_id String,
-    platform Enum('POLYMARKET', 'KALSHI', 'PREDICTIT', 'UNKNOWN'),
+    platform Enum('POLYMARKET', 'KALSHI'),
     outcome Tuple(id UUID, platform_id String, name String, index UInt8),
     bids Array(Tuple(price Decimal64(6), size Decimal64(6))),
     asks Array(Tuple(price Decimal64(6), size Decimal64(6))),
-    timestamp DateTime64(3),
-    indexed_at DateTime64(3)
+    timestamp DateTime64(9),
+    indexed_at DateTime64(9),
+    hash UInt64,
+    state Enum8('VERIFIED' = 1, 'INTERMEDIATE' = 2),
+    continuity Enum8('CONTIGUOUS' = 1, 'RESET' = 2),
+    path_index UInt32
 ) ENGINE = ReplacingMergeTree(indexed_at)
-ORDER BY (market_id, outcome.id, timestamp)
+ORDER BY (market_id, outcome.id, timestamp, hash, state, continuity, path_index)
 PARTITION BY toYYYYMM(timestamp)
 """

{probalytics-0.1.0 → probalytics-0.2.0}/tests/test_clickhouse_integration.py

@@ -133,6 +133,7 @@ def test_fills_dataframe_uses_global_frame_and_market_id_arrays(client: Probalyt
     assert isinstance(fills, pd.DataFrame)
     assert set(fills["market_platform_id"]) == {"0xmarket", "KXBTC-26JUN-T50000"}
     assert set(fills["platform"]) == {"POLYMARKET", "KALSHI"}
+    assert all(value == {} for value in fills["metadata"])
     assert isinstance(fills.loc[0, "outcome"], dict)
     assert set(fills.loc[0, "outcome"]) == {"id", "platform_id", "name", "index"}
     assert isinstance(fills.loc[0, "outcome"]["id"], str)
@@ -169,6 +170,8 @@ def test_orderbook_snapshots_accept_market_platform_id_arrays(client: Probalytic
         end_time="2026-03-15T00:02:00Z",
         platform=["POLYMARKET", "KALSHI"],
         market_platform_id=["0xmarket", "KXBTC-26JUN-T50000"],
+        state="VERIFIED",
+        continuity=["RESET", "CONTIGUOUS"],
         limit=10,
     )
 
@@ -178,6 +181,9 @@ def test_orderbook_snapshots_accept_market_platform_id_arrays(client: Probalytic
     assert isinstance(snapshots.loc[0, "outcome"]["id"], str)
     assert isinstance(snapshots.loc[0, "bids"], list)
     assert isinstance(snapshots.loc[0, "bids"][0], dict)
+    assert set(snapshots["state"]) == {"VERIFIED"}
+    assert set(snapshots["continuity"]) == {"RESET", "CONTIGUOUS"}
+    assert set(snapshots.columns) >= {"indexed_at", "hash", "state", "continuity", "path_index"}
 
 
 def wait_for_clickhouse(host: str, port: int) -> DriverClient:

{probalytics-0.1.0 → probalytics-0.2.0}/tests/test_models.py

@@ -5,9 +5,17 @@ from uuid import UUID
 
 import pytest
 
-from probalytics import ClickHouseClient, Market, ProbalyticsClient
+from probalytics import (
+    ClickHouseClient,
+    Market,
+    MarketType,
+    OrderbookSnapshot,
+    Platform,
+    ProbalyticsClient,
+)
 from probalytics._frames import dataframe_to_frame, normalize_time
 from probalytics._filters import market_filter
+from probalytics.clickhouse import _fill_row
 
 
 def market_payload() -> dict:
@@ -38,6 +46,9 @@ def market_payload() -> dict:
         "end_date": None,
         "reset_at": None,
         "resolution": None,
+        "source_block_number": 0,
+        "source_tx_hash": "",
+        "indexed_at": "2026-03-15T00:00:01Z",
     }
 
 
@@ -51,6 +62,45 @@ def test_market_is_first_class_filter() -> None:
     assert platform == "POLYMARKET"
 
 
+def test_public_enums_only_include_current_values() -> None:
+    assert set(Platform) == {Platform.POLYMARKET, Platform.KALSHI}
+    assert MarketType.PERPETUAL == "PERPETUAL"
+
+    with pytest.raises(ValueError):
+        Platform("PREDICTIT")
+    with pytest.raises(ValueError):
+        MarketType("UNKNOWN")
+
+
+def test_orderbook_snapshot_models_current_quality_fields() -> None:
+    snapshot = OrderbookSnapshot.model_validate(
+        {
+            "market_id": "550e8400-e29b-41d4-a716-446655440000",
+            "market_platform_id": "0xmarket",
+            "platform": "POLYMARKET",
+            "outcome": {
+                "id": "550e8400-e29b-41d4-a716-446655440001",
+                "platform_id": "yes",
+                "name": "Yes",
+                "index": 0,
+            },
+            "bids": [{"price": "0.44", "size": "10"}],
+            "asks": [{"price": "0.46", "size": "8"}],
+            "timestamp": "2026-03-15T00:00:00Z",
+            "indexed_at": "2026-03-15T00:00:00.001Z",
+            "hash": 101,
+            "state": "VERIFIED",
+            "continuity": "RESET",
+            "path_index": 0,
+        }
+    )
+
+    assert snapshot.state == "VERIFIED"
+    assert snapshot.continuity == "RESET"
+    assert snapshot.hash == 101
+    assert snapshot.path_index == 0
+
+
 def test_dataframe_to_polars_preserves_struct() -> None:
     df = dataframe_to_frame(
         {
@@ -150,6 +200,8 @@ def test_market_can_query_orderbook_snapshots_with_bound_client() -> None:
     snapshots = market.orderbook_snapshots(
         start_time="2026-03-15T00:00:00Z",
         end_time="2026-03-15T00:01:00Z",
+        state="VERIFIED",
+        continuity="RESET",
         frame="pandas",
     )
 
@@ -157,6 +209,8 @@ def test_market_can_query_orderbook_snapshots_with_bound_client() -> None:
     assert backend.orderbook_kwargs["market"] is market
     assert backend.orderbook_kwargs["start_time"] == "2026-03-15T00:00:00Z"
     assert backend.orderbook_kwargs["end_time"] == "2026-03-15T00:01:00Z"
+    assert backend.orderbook_kwargs["state"] == "VERIFIED"
+    assert backend.orderbook_kwargs["continuity"] == "RESET"
     assert backend.orderbook_kwargs["frame"] == "pandas"
 
 
@@ -330,6 +384,48 @@ def test_fills_keeps_scalar_filters_as_equals() -> None:
     assert client.client.params["platform"] == "POLYMARKET"
 
 
+def test_fills_selects_provenance_and_metadata() -> None:
+    client = clickhouse_with_recorder()
+
+    client.fills(limit=10)
+
+    assert "source_block_number, source_tx_hash, source_log_index" in client.client.query
+    assert "timestamp, indexed_at, toJSONString(metadata) AS metadata" in client.client.query
+
+
+def test_fill_row_decodes_json_metadata() -> None:
+    row = {
+        "outcome": {
+            "id": "550e8400-e29b-41d4-a716-446655440001",
+            "platform_id": "yes",
+            "name": "Yes",
+            "index": 0,
+        },
+        "metadata": '{"instrument":"BTC","liquidation":false}',
+    }
+
+    assert _fill_row(row)["metadata"] == {"instrument": "BTC", "liquidation": False}
+
+
+def test_orderbook_snapshots_select_quality_fields_and_filters() -> None:
+    client = clickhouse_with_recorder()
+
+    client.orderbook_snapshots(
+        start_time="2026-03-15T00:00:00Z",
+        end_time="2026-03-15T00:01:00Z",
+        state=["VERIFIED", "INTERMEDIATE"],
+        continuity="RESET",
+        limit=10,
+    )
+
+    assert "indexed_at, hash, state, continuity, path_index" in client.client.query
+    assert "state IN %(state)s" in client.client.query
+    assert "continuity = %(continuity)s" in client.client.query
+    assert "ORDER BY timestamp ASC, path_index ASC, indexed_at ASC" in client.client.query
+    assert client.client.params["state"] == ("VERIFIED", "INTERMEDIATE")
+    assert client.client.params["continuity"] == "RESET"
+
+
 def test_clickhouse_rejects_invalid_limits() -> None:
     client = clickhouse_with_recorder()