mdrap 1.0.0__py3-none-any.whl

analytics.py

@@ -0,0 +1,200 @@
+"""
+Analytical Storage & Query Engine module for MDRAP V3.
+Implements Spec §14 — Analytical query and aggregation engine for market data.
+"""
+
+import math
+from typing import Dict, List, Tuple
+
+from models import CanonicalEvent, EventType, QualityStatus
+
+
+class OHLCVAggregator:
+    def __init__(self, interval_s: float = 5.0):
+        self.interval_s = interval_s
+        # Dict key: (instrument_id, bucket_start)
+        # Value: dict holding the raw OHLCV calculations
+        self._buckets: Dict[Tuple[str, float], dict] = {}
+
+    def observe(self, event: CanonicalEvent) -> None:
+        if event.event_type != EventType.TRADE or event.price is None:
+            return
+        if event.quality_status == QualityStatus.INVALID:
+            return
+            
+        bucket_start = float(int(event.exchange_timestamp // self.interval_s) * self.interval_s)
+        key = (event.instrument_id, bucket_start)
+        
+        if key not in self._buckets:
+            self._buckets[key] = {
+                "instrument_id": event.instrument_id,
+                "bucket_start": bucket_start,
+                "interval_s": self.interval_s,
+                "open": event.price,
+                "high": event.price,
+                "low": event.price,
+                "close": event.price,
+                "volume": event.quantity if event.quantity is not None else 0.0,
+                "event_count": 1,
+                "_first_ts": event.exchange_timestamp,
+                "_last_ts": event.exchange_timestamp
+            }
+        else:
+            b = self._buckets[key]
+            first_ts = b.get("_first_ts", b.get("bucket_start", 0.0))
+            last_ts = b.get("_last_ts", b.get("bucket_start", 0.0))
+            if event.exchange_timestamp < first_ts:
+                b["open"] = event.price
+                b["_first_ts"] = event.exchange_timestamp
+            if event.exchange_timestamp >= last_ts:
+                b["close"] = event.price
+                b["_last_ts"] = event.exchange_timestamp
+                
+            b["high"] = max(b["high"], event.price)
+            b["low"] = min(b["low"], event.price)
+            if event.quantity is not None:
+                b["volume"] += event.quantity
+            b["event_count"] += 1
+
+    def _format_candle(self, b: dict) -> dict:
+        return {
+            "instrument_id": b["instrument_id"],
+            "bucket_start": b["bucket_start"],
+            "interval_s": b["interval_s"],
+            "open": b["open"],
+            "high": b["high"],
+            "low": b["low"],
+            "close": b["close"],
+            "volume": b["volume"],
+            "event_count": b["event_count"]
+        }
+
+    def candles(self) -> list[dict]:
+        sorted_keys = sorted(self._buckets.keys())
+        return [self._format_candle(self._buckets[k]) for k in sorted_keys]
+
+    def candles_for(self, instrument_id: str) -> list[dict]:
+        res = []
+        for k in sorted(self._buckets.keys()):
+            if k[0] == instrument_id:
+                res.append(self._format_candle(self._buckets[k]))
+        return res
+
+
+class SpreadAnalyzer:
+    def __init__(self):
+        self._stats: Dict[str, dict] = {}
+
+    def observe(self, event: CanonicalEvent) -> None:
+        if event.event_type != EventType.QUOTE or event.bid_price is None or event.ask_price is None:
+            return
+        if not math.isfinite(event.bid_price) or not math.isfinite(event.ask_price):
+            return
+            
+        spread = event.ask_price - event.bid_price
+        crossed = 1 if event.bid_price > event.ask_price else 0
+        
+        if event.instrument_id not in self._stats:
+            self._stats[event.instrument_id] = {
+                "quote_count": 1,
+                "sum_spread": spread,
+                "min_spread": spread,
+                "max_spread": spread,
+                "crossed_count": crossed
+            }
+        else:
+            s = self._stats[event.instrument_id]
+            s["quote_count"] += 1
+            s["sum_spread"] += spread
+            s["min_spread"] = min(s["min_spread"], spread)
+            s["max_spread"] = max(s["max_spread"], spread)
+            s["crossed_count"] += crossed
+
+    def summary(self) -> list[dict]:
+        res = []
+        for instr, s in self._stats.items():
+            count = s["quote_count"]
+            mean_spread = s["sum_spread"] / count if count > 0 else 0.0
+            crossed_pct = (s["crossed_count"] / count * 100) if count > 0 else 0.0
+            res.append({
+                "instrument_id": instr,
+                "quote_count": count,
+                "mean_spread": mean_spread,
+                "min_spread": s["min_spread"],
+                "max_spread": s["max_spread"],
+                "crossed_count": s["crossed_count"],
+                "crossed_pct": crossed_pct
+            })
+        return res
+
+
+class VolatilityTracker:
+    def __init__(self, window: int = 100):
+        self.window = window
+        self._stats: Dict[str, dict] = {}
+
+    def observe(self, event: CanonicalEvent) -> None:
+        if event.event_type != EventType.TRADE or event.price is None:
+            return
+        if not math.isfinite(event.price) or event.price <= 0:
+            return
+            
+        p = event.price
+
+        instr = event.instrument_id
+        
+        if instr not in self._stats:
+            self._stats[instr] = {
+                "count": 1,
+                "mean": p,
+                "M2": 0.0,
+                "min_price": p,
+                "max_price": p
+            }
+        else:
+            s = self._stats[instr]
+            s["count"] += 1
+            delta = p - s["mean"]
+            s["mean"] += delta / s["count"]
+            delta2 = p - s["mean"]
+            s["M2"] += delta * delta2
+            s["min_price"] = min(s["min_price"], p)
+            s["max_price"] = max(s["max_price"], p)
+
+    def summary(self) -> list[dict]:
+        res = []
+        for instr, s in self._stats.items():
+            count = s["count"]
+            mean = s["mean"]
+            std_dev = math.sqrt(s["M2"] / count) if count > 0 else 0.0
+            price_range_pct = ((s["max_price"] - s["min_price"]) / mean * 100) if mean > 0 else 0.0
+            
+            res.append({
+                "instrument_id": instr,
+                "trade_count": count,
+                "mean_price": mean,
+                "std_dev": std_dev,
+                "min_price": s["min_price"],
+                "max_price": s["max_price"],
+                "price_range_pct": price_range_pct
+            })
+        return res
+
+
+class MarketAnalytics:
+    def __init__(self, ohlcv_interval_s: float = 5.0, volatility_window: int = 100):
+        self.ohlcv = OHLCVAggregator(interval_s=ohlcv_interval_s)
+        self.spreads = SpreadAnalyzer()
+        self.volatility = VolatilityTracker(window=volatility_window)
+
+    def observe(self, event: CanonicalEvent) -> None:
+        self.ohlcv.observe(event)
+        self.spreads.observe(event)
+        self.volatility.observe(event)
+
+    def full_summary(self) -> dict:
+        return {
+            "ohlcv": self.ohlcv.candles(),
+            "spreads": self.spreads.summary(),
+            "volatility": self.volatility.summary()
+        }

archive.py

@@ -0,0 +1,154 @@
+"""
+Immutable Raw Event Archive module.
+
+Implements Spec §6.10 and §19 — immutable raw event persistence.
+This is a write-ahead JSONL archive that captures every raw event BEFORE processing, 
+partitioned by date and source.
+"""
+
+import json
+import os
+import time
+from typing import Iterator, Optional, Any, Dict, List
+
+from models import RawEvent
+
+
+class RawArchive:
+    def __init__(self, base_dir: str = 'data/raw_archive', buffer_size: int = 500):
+        self.base_dir = base_dir
+        self.buffer_size = buffer_size
+        self._buffer: list[RawEvent] = []
+        self._file_handles: dict[str, Any] = {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def write(self, raw: RawEvent) -> None:
+        self._buffer.append(raw)
+        if len(self._buffer) >= self.buffer_size:
+            self.flush()
+
+    def flush(self) -> None:
+        if not self._buffer:
+            return
+
+        grouped: dict[str, list[str]] = {}
+        for raw in self._buffer:
+            # Format receive_timestamp to YYYY-MM-DD for partitioning
+            date_str = time.strftime('%Y-%m-%d', time.gmtime(raw.receive_timestamp))
+            dir_path = os.path.join(self.base_dir, date_str)
+            file_path = os.path.join(dir_path, f"{raw.source}.jsonl")
+
+            os.makedirs(dir_path, exist_ok=True)
+
+            if file_path not in grouped:
+                grouped[file_path] = []
+
+            # Line is a JSON object
+            line = json.dumps({
+                "raw_id": raw.raw_id,
+                "source": raw.source,
+                "payload": raw.payload,
+                "receive_timestamp": raw.receive_timestamp
+            })
+            grouped[file_path].append(line)
+
+        for file_path, lines in grouped.items():
+            if file_path not in self._file_handles:
+                self._file_handles[file_path] = open(file_path, 'a', encoding='utf-8')
+
+            handle = self._file_handles[file_path]
+            for line in lines:
+                handle.write(line + '\n')
+            handle.flush()
+
+        self._buffer.clear()
+
+    def close(self) -> None:
+        self.flush()
+        for handle in self._file_handles.values():
+            handle.close()
+        self._file_handles.clear()
+
+    def stats(self) -> dict:
+        total_events = 0
+        dates = set()
+        sources = set()
+        size_bytes = 0
+
+        if not os.path.exists(self.base_dir):
+            return {
+                "total_events": 0,
+                "dates": [],
+                "sources": [],
+                "size_bytes": 0
+            }
+
+        for root, _, files in os.walk(self.base_dir):
+            for file in files:
+                if file.endswith('.jsonl'):
+                    file_path = os.path.join(root, file)
+                    date_dir = os.path.basename(root)
+                    dates.add(date_dir)
+                    
+                    source = file[:-6]  # remove .jsonl
+                    sources.add(source)
+
+                    size_bytes += os.path.getsize(file_path)
+
+                    # count lines
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        for _ in f:
+                            total_events += 1
+
+        return {
+            "total_events": total_events,
+            "dates": sorted(list(dates)),
+            "sources": sorted(list(sources)),
+            "size_bytes": size_bytes
+        }
+
+
+def replay(base_dir: str, date: Optional[str] = None, source: Optional[str] = None) -> Iterator[RawEvent]:
+    """
+    Yields RawEvent objects from archived JSONL files.
+    """
+    if not os.path.exists(base_dir):
+        return
+
+    # Get all date directories sorted
+    date_dirs = sorted([d for d in os.listdir(base_dir) if os.path.isdir(os.path.join(base_dir, d))])
+
+    if date:
+        date_dirs = [d for d in date_dirs if d == date]
+
+    for date_dir in date_dirs:
+        dir_path = os.path.join(base_dir, date_dir)
+
+        # Get all jsonl files
+        files = sorted([f for f in os.listdir(dir_path) if f.endswith('.jsonl')])
+
+        if source:
+            target_file = f"{source}.jsonl"
+            files = [f for f in files if f == target_file]
+
+        for file in files:
+            file_path = os.path.join(dir_path, file)
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if line.strip():
+                        try:
+                            data = json.loads(line)
+                            yield RawEvent(
+                                raw_id=data["raw_id"],
+                                source=data["source"],
+                                payload=data["payload"],
+                                receive_timestamp=data["receive_timestamp"]
+                            )
+                        except (json.JSONDecodeError, KeyError):
+                            # Skip corrupted/truncated archive lines without crashing replay
+                            continue

async_storage.py

@@ -0,0 +1,261 @@
+"""
+Dedicated Asynchronous Storage Worker for MDRAP.
+
+Decouples synchronous SQLite and DuckDB disk commits (fsync) from the real-time
+market data processing loop. Canonical events, quarantine tuples, and lineage
+records are enqueued into a lock-free SPSC ring buffer and flushed in batches
+by a dedicated background worker thread.
+
+Zero disk I/O pauses on the tick broadcasting hot path.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import Any, List, Optional, Tuple, Union
+
+from models import CanonicalEvent
+from spsc_ring import SPSCRingBuffer
+from storage import Store
+
+logger = logging.getLogger(__name__)
+
+# Action tags for storage worker
+OP_CANONICAL = 1
+OP_QUARANTINE = 2
+OP_LINEAGE = 3
+OP_HEALTH = 4
+OP_FLUSH_BARRIER = 5
+
+
+class StorageCommand:
+    __slots__ = ("op_type", "payload")
+
+    def __init__(self, op_type: int, payload: Any):
+        self.op_type = op_type
+        self.payload = payload
+
+
+class AsyncStorageWorker:
+    """
+    Background worker thread that drains persistence commands from an SPSC ring buffer
+    and commits them to SQLite in optimized batches.
+    """
+
+    def __init__(
+        self,
+        store: Store,
+        queue_capacity: int = 65536,
+        batch_size: int = 2000,
+        flush_interval_s: float = 0.25,
+        duck_path: Optional[str] = None,
+    ):
+        self.store = store
+        self.batch_size = batch_size
+        self.flush_interval_s = flush_interval_s
+        self.duck_path = duck_path
+
+        self._queue: SPSCRingBuffer[StorageCommand] = SPSCRingBuffer(capacity=queue_capacity)
+        self._running = False
+        self._thread: Optional[threading.Thread] = None
+        self._flush_lock = threading.Lock()
+        self._flush_complete_event = threading.Event()
+
+        # Telemetry counters
+        self._total_canonical = 0
+        self._total_quarantine = 0
+        self._total_lineage = 0
+        self._total_commits = 0
+        self._last_commit_ms = 0.0
+        self._max_commit_ms = 0.0
+        self._total_dropped = 0
+
+    @property
+    def queue(self) -> SPSCRingBuffer[StorageCommand]:
+        return self._queue
+
+    def start(self) -> None:
+        """Start the background storage thread."""
+        if self._running:
+            return
+        self._running = True
+        self._thread = threading.Thread(target=self._worker_loop, name="mdrap-async-storage", daemon=True)
+        self._thread.start()
+
+    def stop(self, timeout: float = 5.0) -> None:
+        """Gracefully stop worker and drain all pending storage writes."""
+        if not self._running:
+            return
+        self._running = False
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=timeout)
+
+        # Final synchronous flush of any residual queue items
+        self._drain_and_commit(force_all=True)
+
+    def write_canonical(self, event: CanonicalEvent) -> bool:
+        """Enqueue a validated canonical event for background SQLite storage."""
+        cmd = StorageCommand(OP_CANONICAL, event)
+        ok = self._queue.offer(cmd)
+        if not ok:
+            self._total_dropped += 1
+        return ok
+
+    def write_quarantine(self, row: tuple) -> bool:
+        """Enqueue an invalid/suspicious quarantine record."""
+        cmd = StorageCommand(OP_QUARANTINE, row)
+        ok = self._queue.offer(cmd)
+        if not ok:
+            self._total_dropped += 1
+        return ok
+
+    def write_lineage(self, row: tuple) -> bool:
+        """Enqueue an audit lineage trace."""
+        cmd = StorageCommand(OP_LINEAGE, row)
+        ok = self._queue.offer(cmd)
+        if not ok:
+            self._total_dropped += 1
+        return ok
+
+    def write_health(self, rows: list) -> bool:
+        """Enqueue source health telemetry."""
+        cmd = StorageCommand(OP_HEALTH, rows)
+        ok = self._queue.offer(cmd)
+        if not ok:
+            self._total_dropped += 1
+        return ok
+
+    def flush(self, timeout: float = 2.0) -> None:
+        """Synchronously wait until all currently queued events are committed to disk."""
+        if not self._running:
+            self._drain_and_commit(force_all=True)
+            return
+
+        with self._flush_lock:
+            self._flush_complete_event.clear()
+            barrier = StorageCommand(OP_FLUSH_BARRIER, self._flush_complete_event)
+            while not self._queue.offer(barrier):
+                time.sleep(0.0005)
+            self._flush_complete_event.wait(timeout=timeout)
+
+    def _worker_loop(self) -> None:
+        """Main background loop draining SPSC queue and committing in batches."""
+        last_flush = time.time()
+        canonical_batch: List[CanonicalEvent] = []
+        quarantine_batch: List[tuple] = []
+        lineage_batch: List[tuple] = []
+        health_batch: List[tuple] = []
+
+        while self._running:
+            cmd = self._queue.poll()
+            if cmd is not None:
+                if cmd.op_type == OP_CANONICAL:
+                    canonical_batch.append(cmd.payload)
+                elif cmd.op_type == OP_QUARANTINE:
+                    quarantine_batch.append(cmd.payload)
+                elif cmd.op_type == OP_LINEAGE:
+                    lineage_batch.append(cmd.payload)
+                elif cmd.op_type == OP_HEALTH:
+                    health_batch.extend(cmd.payload)
+                elif cmd.op_type == OP_FLUSH_BARRIER:
+                    # Flush immediately on barrier
+                    self._commit_batches(canonical_batch, quarantine_batch, lineage_batch, health_batch)
+                    last_flush = time.time()
+                    if isinstance(cmd.payload, threading.Event):
+                        cmd.payload.set()
+                    continue
+
+            now = time.time()
+            total_pending = len(canonical_batch) + len(quarantine_batch) + len(lineage_batch)
+            time_elapsed = (now - last_flush) >= self.flush_interval_s
+
+            if total_pending >= self.batch_size or (total_pending > 0 and time_elapsed):
+                self._commit_batches(canonical_batch, quarantine_batch, lineage_batch, health_batch)
+                last_flush = now
+            elif cmd is None:
+                # Yield CPU briefly if queue was empty
+                time.sleep(0.001)
+
+        # Drain any residual items on exit
+        self._commit_batches(canonical_batch, quarantine_batch, lineage_batch, health_batch)
+
+    def _commit_batches(
+        self,
+        canonical_batch: List[CanonicalEvent],
+        quarantine_batch: List[tuple],
+        lineage_batch: List[tuple],
+        health_batch: List[tuple],
+    ) -> None:
+        if not canonical_batch and not quarantine_batch and not lineage_batch and not health_batch:
+            return
+
+        t0 = time.perf_counter()
+        try:
+            if canonical_batch:
+                self.store.write_canonical_batch(canonical_batch)
+                self._total_canonical += len(canonical_batch)
+                canonical_batch.clear()
+
+            if quarantine_batch:
+                self.store.write_quarantine_batch(quarantine_batch)
+                self._total_quarantine += len(quarantine_batch)
+                quarantine_batch.clear()
+
+            if lineage_batch:
+                self.store.write_lineage_batch(lineage_batch)
+                self._total_lineage += len(lineage_batch)
+                lineage_batch.clear()
+
+            if health_batch:
+                self.store.upsert_source_health(health_batch)
+                health_batch.clear()
+
+            self.store.commit()
+            self._total_commits += 1
+            dur_ms = (time.perf_counter() - t0) * 1000.0
+            self._last_commit_ms = round(dur_ms, 2)
+            if dur_ms > self._max_commit_ms:
+                self._max_commit_ms = round(dur_ms, 2)
+        except Exception as e:
+            logger.error(f"Async storage batch commit failed: {e}")
+
+    def _drain_and_commit(self, force_all: bool = True) -> None:
+        """Synchronous drain of the queue for shutdown or offline testing."""
+        canonical_batch: List[CanonicalEvent] = []
+        quarantine_batch: List[tuple] = []
+        lineage_batch: List[tuple] = []
+        health_batch: List[tuple] = []
+
+        while True:
+            cmd = self._queue.poll()
+            if cmd is None:
+                break
+            if cmd.op_type == OP_CANONICAL:
+                canonical_batch.append(cmd.payload)
+            elif cmd.op_type == OP_QUARANTINE:
+                quarantine_batch.append(cmd.payload)
+            elif cmd.op_type == OP_LINEAGE:
+                lineage_batch.append(cmd.payload)
+            elif cmd.op_type == OP_HEALTH:
+                health_batch.extend(cmd.payload)
+            elif cmd.op_type == OP_FLUSH_BARRIER:
+                if isinstance(cmd.payload, threading.Event):
+                    cmd.payload.set()
+
+        self._commit_batches(canonical_batch, quarantine_batch, lineage_batch, health_batch)
+
+    def stats(self) -> dict:
+        """Return operational telemetry."""
+        return {
+            "is_running": self._running,
+            "queue_size": self._queue.size(),
+            "queue_capacity": self._queue.capacity,
+            "total_canonical": self._total_canonical,
+            "total_quarantine": self._total_quarantine,
+            "total_lineage": self._total_lineage,
+            "total_commits": self._total_commits,
+            "last_commit_ms": self._last_commit_ms,
+            "max_commit_ms": self._max_commit_ms,
+            "total_dropped": self._total_dropped,
+        }