mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/stream_connector.py

@@ -0,0 +1,418 @@
+"""Closed declarative connector configurations for recorded stream capture.
+
+Recorder code is generic; a source is a closed declarative configuration that is digested and
+bound into capture evidence.  This module holds that configuration contract and the connector
+configurations this build knows about.
+
+Nothing here opens a connection.  ``endpoint_url`` and ``subscribe_template`` are *grammar*: the
+address a trusted Recorder would hold and the exact subscription message it would send.  This
+build records the grammar, checks it, and digests it; the socket belongs to a trusted Recorder
+role that does not exist yet, and the deterministic path is never allowed to open one.  The
+validation below is therefore about whether a configuration is well formed and bounded, never
+about whether an address answers.
+
+The subscription member set is not part of the configuration.  Which markets a capture
+subscribes to is a declared bounded set pinned in the Recipe's canonical query, exactly as
+``public.https.collection`` pins its members: source content cannot add a member.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+STREAM_CONNECTOR_VERSION = "harness-stream-connector.v1"
+
+# Every message this build will admit is classified into exactly one of these three sets by the
+# connector's ``message_type_field``.  A message whose type is in none of them is refused rather
+# than guessed at: an unclassified message is either a venue change or a wrong connector, and
+# both deserve a stop.
+MESSAGE_CLASSES = ("snapshot", "delta", "ignored")
+
+# The three gap-repair modes cover the known universe of venue behaviour.  ``none`` is not a
+# failure to configure: for an event-only stream the gap really is permanent, and saying so is
+# the honest answer.
+GAP_REPAIR_MODES = frozenset({"snapshot_on_subscribe", "cursor_resume", "none"})
+
+# Where an event's identity comes from.  ``provider`` extracts a venue-assigned ID; ``derived``
+# assigns ``{epoch}:{counter}`` because the venue offers nothing stable.
+EVENT_ID_RULES = frozenset({"provider", "derived"})
+
+# How to read the event-time field.  The interpretation is declared, never sniffed: a field of
+# digits is milliseconds or seconds depending on the venue, and guessing wrong moves every event
+# by a factor of a thousand.
+EVENT_TIME_KINDS: Mapping[str, re.Pattern[str]] = MappingProxyType(
+    {
+        "rfc3339_utc": re.compile(
+            r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z"
+        ),
+        "epoch_milliseconds_text": re.compile(r"[0-9]{1,15}"),
+        "epoch_seconds_text": re.compile(r"[0-9]{1,12}"),
+    }
+)
+
+_CONNECTOR_ID = re.compile(r"[a-z][a-z0-9]*(?:\.[a-z0-9][a-z0-9-]*)+")
+# The stream identifier addresses one directory beneath the capture root, so it is a strict
+# single path component: no dot, no separator, nothing that could climb out of the root.
+_STREAM_ID = re.compile(r"[a-z][a-z0-9-]{0,63}")
+# A recorded address names a host, never a literal one.  The trailing alphabetic label is what
+# refuses ``127.0.0.1`` and its friends: an address that is already a number answers to no naming
+# policy, so a Recorder must not be pointed at one from a configuration.
+_HOSTNAME = re.compile(
+    r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,63}"
+)
+_MESSAGE_FIELD = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,63}")
+_MESSAGE_TYPE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")
+# A capture root reserves this name for the durable per-source bookmark ledger, so no stream may
+# claim it as its own directory.
+RESERVED_STREAM_IDS = frozenset({"bookmarks", "batches"})
+
+# The hard bound on one sealed batch, mirroring ``stream.MAX_BATCH_EVENTS``.  It is restated as
+# a bound on the *configuration* so an unbuildable connector is refused where it is written down
+# rather than at the first rotation.
+MAX_CONFIGURED_BATCH_EVENTS = 100_000
+MAX_CONFIGURED_BATCH_BYTES = 64 * 1024 * 1024
+MAX_CONFIGURED_BATCH_SECONDS = 24 * 60 * 60
+
+
+@dataclass(frozen=True)
+class StreamConnectorConfig:
+    """One venue's closed capture grammar, digested and bound into capture evidence."""
+
+    connector_id: str
+    stream_id: str
+    endpoint_url: str
+    subscribe_template: Mapping[str, Any]
+    subscription_member_key: str
+    message_type_field: str
+    snapshot_message_types: tuple[str, ...]
+    delta_message_types: tuple[str, ...]
+    ignored_message_types: tuple[str, ...]
+    event_id_rule: str
+    provider_event_id_field: str | None
+    event_time_field: str
+    event_time_kind: str
+    gap_repair: str
+    heartbeat_seconds: int
+    reconnect_backoff_seconds: int
+    max_batch_events: int
+    max_batch_bytes: int
+    max_batch_seconds: int
+    data_format: str = "ndjson"
+    schema_version: str = STREAM_CONNECTOR_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != STREAM_CONNECTOR_VERSION:
+            raise SourceContractError(
+                "STREAM_CONNECTOR_VERSION",
+                "connector.schema_version",
+                "is unsupported",
+            )
+        _matched(self.connector_id, _CONNECTOR_ID, "connector.connector_id")
+        _matched(self.stream_id, _STREAM_ID, "connector.stream_id")
+        if self.stream_id in RESERVED_STREAM_IDS:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.stream_id",
+                f"must not be one of the reserved capture names {sorted(RESERVED_STREAM_IDS)}",
+            )
+        _subscription_endpoint(self.endpoint_url)
+        _subscribe_template(self.subscribe_template, self.subscription_member_key)
+        _matched(self.message_type_field, _MESSAGE_FIELD, "connector.message_type_field")
+        _matched(self.event_time_field, _MESSAGE_FIELD, "connector.event_time_field")
+        self._validate_message_types()
+        if self.event_id_rule not in EVENT_ID_RULES:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.event_id_rule",
+                f"must be one of {sorted(EVENT_ID_RULES)}",
+            )
+        if (self.provider_event_id_field is None) != (self.event_id_rule == "derived"):
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.provider_event_id_field",
+                "must name a field exactly when the event-ID rule extracts a provider ID",
+            )
+        if self.provider_event_id_field is not None:
+            _matched(
+                self.provider_event_id_field,
+                _MESSAGE_FIELD,
+                "connector.provider_event_id_field",
+            )
+        if self.event_time_kind not in EVENT_TIME_KINDS:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.event_time_kind",
+                f"must be one of {sorted(EVENT_TIME_KINDS)}",
+            )
+        if self.gap_repair not in GAP_REPAIR_MODES:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.gap_repair",
+                f"must be one of {sorted(GAP_REPAIR_MODES)}",
+            )
+        if self.gap_repair == "snapshot_on_subscribe" and not self.snapshot_message_types:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.gap_repair",
+                "snapshot repair requires at least one snapshot message type",
+            )
+        if self.data_format not in {"json", "ndjson"}:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.data_format",
+                "must be json or ndjson",
+            )
+        self._validate_bounds()
+
+    def _validate_message_types(self) -> None:
+        groups = (
+            ("snapshot_message_types", self.snapshot_message_types),
+            ("delta_message_types", self.delta_message_types),
+            ("ignored_message_types", self.ignored_message_types),
+        )
+        seen: set[str] = set()
+        for name, values in groups:
+            if not isinstance(values, tuple) or len(values) > 64:
+                raise SourceContractError(
+                    "STREAM_CONNECTOR",
+                    f"connector.{name}",
+                    "must be a bounded immutable tuple",
+                )
+            for value in values:
+                _matched(value, _MESSAGE_TYPE, f"connector.{name}")
+                if value in seen:
+                    raise SourceContractError(
+                        "STREAM_CONNECTOR",
+                        f"connector.{name}",
+                        f"message type {value!r} is classified more than once",
+                    )
+                seen.add(value)
+        if not self.snapshot_message_types and not self.delta_message_types:
+            raise SourceContractError(
+                "STREAM_CONNECTOR",
+                "connector.delta_message_types",
+                "a connector that admits no message at all captures nothing",
+            )
+
+    def _validate_bounds(self) -> None:
+        bounds = (
+            ("heartbeat_seconds", self.heartbeat_seconds, 1, 3_600),
+            ("reconnect_backoff_seconds", self.reconnect_backoff_seconds, 1, 3_600),
+            ("max_batch_events", self.max_batch_events, 1, MAX_CONFIGURED_BATCH_EVENTS),
+            ("max_batch_bytes", self.max_batch_bytes, 1, MAX_CONFIGURED_BATCH_BYTES),
+            ("max_batch_seconds", self.max_batch_seconds, 1, MAX_CONFIGURED_BATCH_SECONDS),
+        )
+        for name, value, low, high in bounds:
+            if type(value) is not int or not low <= value <= high:
+                raise SourceContractError(
+                    "STREAM_CONNECTOR",
+                    f"connector.{name}",
+                    f"must be an integer between {low} and {high}",
+                )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def message_class(self, message_type: str) -> str:
+        """Return which of the three closed classes a message type belongs to.
+
+        Raises rather than defaulting.  An unclassified message type means the venue changed
+        its message set or the wrong connector was chosen, and both are stops.
+        """
+
+        if message_type in self.snapshot_message_types:
+            return "snapshot"
+        if message_type in self.delta_message_types:
+            return "delta"
+        if message_type in self.ignored_message_types:
+            return "ignored"
+        raise SourceContractError(
+            "STREAM_MESSAGE_TYPE",
+            "recording.message",
+            f"message type {message_type!r} is not classified by connector {self.connector_id}",
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "connector_id": self.connector_id,
+            "stream_id": self.stream_id,
+            "endpoint_url": self.endpoint_url,
+            "subscribe_template": dict(self.subscribe_template),
+            "subscription_member_key": self.subscription_member_key,
+            "message_type_field": self.message_type_field,
+            "snapshot_message_types": list(self.snapshot_message_types),
+            "delta_message_types": list(self.delta_message_types),
+            "ignored_message_types": list(self.ignored_message_types),
+            "event_id_rule": self.event_id_rule,
+            "provider_event_id_field": self.provider_event_id_field,
+            "event_time_field": self.event_time_field,
+            "event_time_kind": self.event_time_kind,
+            "gap_repair": self.gap_repair,
+            "heartbeat_seconds": self.heartbeat_seconds,
+            "reconnect_backoff_seconds": self.reconnect_backoff_seconds,
+            "max_batch_events": self.max_batch_events,
+            "max_batch_bytes": self.max_batch_bytes,
+            "max_batch_seconds": self.max_batch_seconds,
+            "data_format": self.data_format,
+        }
+
+
+def _matched(value: Any, pattern: re.Pattern[str], path: str) -> None:
+    if not isinstance(value, str) or pattern.fullmatch(value) is None:
+        raise SourceContractError("STREAM_CONNECTOR", path, "does not match its exact grammar")
+
+
+def _subscription_endpoint(value: Any) -> None:
+    """Check the recorded address is a well-formed credential-free secure socket address.
+
+    Grammar only.  Nothing in this build resolves, connects to, or otherwise touches the
+    address; the check exists so a malformed or credential-bearing address is refused where it
+    is written down rather than carried into a Recorder that would.
+    """
+
+    if not isinstance(value, str) or not 1 <= len(value) <= 2_048:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must be a bounded non-empty address",
+        )
+    parsed = urlsplit(value)
+    if parsed.scheme != "wss":
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must name the wss scheme; a Recorder holds no other kind of connection",
+        )
+    if parsed.username is not None or parsed.password is not None:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must carry no inline credential",
+        )
+    if parsed.query or parsed.fragment:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must carry no query or fragment; subscription is a message, not an address",
+        )
+    try:
+        port = parsed.port
+    except ValueError:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "carries an unreadable port",
+        ) from None
+    if port not in {None, 443}:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must use the default secure port",
+        )
+    hostname = parsed.hostname
+    if hostname is None or _HOSTNAME.fullmatch(hostname) is None:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must name a plain lowercase dotted host, not a literal address",
+        )
+    if parsed.path and re.fullmatch(r"(/[A-Za-z0-9._~-]+)+", parsed.path) is None:
+        raise SourceContractError(
+            "STREAM_ENDPOINT",
+            "connector.endpoint_url",
+            "must carry a plain unescaped path",
+        )
+
+
+def _subscribe_template(value: Any, member_key: Any) -> None:
+    """Check the subscription message is a flat closed template with one member slot."""
+
+    if not isinstance(value, Mapping) or not 1 <= len(value) <= 16:
+        raise SourceContractError(
+            "STREAM_SUBSCRIBE",
+            "connector.subscribe_template",
+            "must be a small non-empty mapping",
+        )
+    for key, item in value.items():
+        _matched(key, _MESSAGE_FIELD, "connector.subscribe_template")
+        if not isinstance(item, str | list) or (isinstance(item, list) and item):
+            raise SourceContractError(
+                "STREAM_SUBSCRIBE",
+                "connector.subscribe_template",
+                "values must be fixed text or the one empty member slot",
+            )
+    if not isinstance(member_key, str) or member_key not in value:
+        raise SourceContractError(
+            "STREAM_SUBSCRIBE",
+            "connector.subscription_member_key",
+            "must name a key of the subscription message",
+        )
+    if not isinstance(value[member_key], list):
+        raise SourceContractError(
+            "STREAM_SUBSCRIBE",
+            "connector.subscription_member_key",
+            "must name the empty member slot the Recipe's declared set fills",
+        )
+
+
+# Polymarket's public market channel.  Credential-free, and the first target because no public
+# archive of its order-book depth exists: whoever captures it from now owns history nobody else
+# can reconstruct.
+#
+# The venue re-sends a full ``book`` snapshot on every subscribe, so a new connection repairs
+# its own gap and the epoch is self-repairing.  There are no venue sequence numbers, so event
+# identity is Recorder-assigned ``{epoch}:{counter}``.
+#
+# Endpoint and message shapes are recorded here from the venue's published documentation and are
+# not verified against the live venue by this build, which reaches no network at all.
+POLYMARKET_CLOB_MARKET = StreamConnectorConfig(
+    connector_id="polymarket.clob-market",
+    stream_id="polymarket-clob-market",
+    endpoint_url="wss://ws-subscriptions-clob.polymarket.com/ws/market",
+    subscribe_template={"type": "market", "assets_ids": []},
+    subscription_member_key="assets_ids",
+    message_type_field="event_type",
+    snapshot_message_types=("book",),
+    delta_message_types=("price_change", "tick_size_change", "last_trade_price"),
+    ignored_message_types=("pong",),
+    event_id_rule="derived",
+    provider_event_id_field=None,
+    event_time_field="timestamp",
+    event_time_kind="epoch_milliseconds_text",
+    gap_repair="snapshot_on_subscribe",
+    heartbeat_seconds=10,
+    reconnect_backoff_seconds=5,
+    max_batch_events=5_000,
+    max_batch_bytes=8 * 1024 * 1024,
+    max_batch_seconds=300,
+)
+
+
+KNOWN_STREAM_CONNECTORS: Mapping[str, StreamConnectorConfig] = MappingProxyType(
+    {POLYMARKET_CLOB_MARKET.connector_id: POLYMARKET_CLOB_MARKET}
+)
+
+STREAM_CONNECTOR_IDS: tuple[str, ...] = tuple(sorted(KNOWN_STREAM_CONNECTORS))
+
+
+def stream_connector(connector_id: str) -> StreamConnectorConfig:
+    """Return the one connector configuration this build knows by that name."""
+
+    try:
+        return KNOWN_STREAM_CONNECTORS[connector_id]
+    except KeyError:
+        raise SourceContractError(
+            "STREAM_CONNECTOR_UNKNOWN",
+            "connector_id",
+            f"connector {connector_id!r} is not one of {list(STREAM_CONNECTOR_IDS)}",
+        ) from None