tt-data-store 0.1.0__py3-none-any.whl

tt_data_store/__init__.py

@@ -0,0 +1,14 @@
+"""Canonical market data archive library."""
+
+from tt_data_store.store import MarketStore
+from tt_data_store.remote import RemoteConfig, SyncConflict, TransferResult
+from tt_data_store.validation import ValidationError, ValidationResult
+
+__all__ = [
+    "MarketStore",
+    "RemoteConfig",
+    "SyncConflict",
+    "TransferResult",
+    "ValidationError",
+    "ValidationResult",
+]

tt_data_store/atomic.py

@@ -0,0 +1,22 @@
+"""Atomic Parquet replacement (doc 06)."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def atomic_replace(temp_path: Path, target_path: Path) -> None:
+    """Replace target with temp via sync + os.replace.
+
+    Temporary files live beside the target as ``.{name}.tmp`` (doc 06).
+    """
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(temp_path, "rb") as f:
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(temp_path, target_path)
+
+
+def temp_path_for(target_path: Path) -> Path:
+    return target_path.parent / f".{target_path.name}.tmp"

tt_data_store/calendar.py

@@ -0,0 +1,98 @@
+"""Market calendar sessions → expected timestamps (doc 08)."""
+
+from __future__ import annotations
+
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from tt_data_store.schema import TZ
+
+IST = ZoneInfo(TZ)
+
+CALENDAR_SCHEMA = pa.schema(
+    [
+        pa.field("date", pa.date32(), nullable=False),
+        pa.field("market", pa.string(), nullable=False),
+        pa.field("session_start", pa.timestamp("us", tz=TZ), nullable=True),
+        pa.field("session_end", pa.timestamp("us", tz=TZ), nullable=True),
+        pa.field("event", pa.string(), nullable=True),
+    ]
+)
+
+
+def calendar_path(root: Path, market: str = "nse") -> Path:
+    return root / "reference" / "calendar" / f"{market.lower()}.parquet"
+
+
+def load_sessions(
+    root: Path,
+    *,
+    market: str = "NSE",
+    start: date | None = None,
+    end: date | None = None,
+) -> list[tuple[datetime, datetime]]:
+    path = calendar_path(root, market="nse" if market.upper() == "NSE" else market.lower())
+    if not path.exists():
+        return []
+    table = pq.read_table(path)
+    sessions: list[tuple[datetime, datetime]] = []
+    for i in range(table.num_rows):
+        mkt = table.column("market")[i].as_py()
+        if str(mkt).upper() != market.upper():
+            continue
+        d = table.column("date")[i].as_py()
+        if start is not None and d < start:
+            continue
+        if end is not None and d > end:
+            continue
+        s_start = table.column("session_start")[i].as_py()
+        s_end = table.column("session_end")[i].as_py()
+        if s_start is None or s_end is None:
+            # Holiday / non-trading row: no expected timestamps
+            continue
+        if s_start.tzinfo is None:
+            s_start = s_start.replace(tzinfo=IST)
+        else:
+            s_start = s_start.astimezone(IST)
+        if s_end.tzinfo is None:
+            s_end = s_end.replace(tzinfo=IST)
+        else:
+            s_end = s_end.astimezone(IST)
+        sessions.append((s_start, s_end))
+    return sessions
+
+
+def expected_timestamps(
+    sessions: list[tuple[datetime, datetime]],
+) -> list[datetime]:
+    """Generate 1-minute timestamps for half-open session [start, end).
+
+    Candles use timestamp = start of the minute (doc 02). If the market closes
+    at ``session_end`` (e.g. 15:30), the last expected bar is ``session_end - 1m``
+    (e.g. 15:29). ``session_end`` itself is not an expected candle timestamp.
+    """
+    out: list[datetime] = []
+    for start, end in sessions:
+        ts = start
+        while ts < end:
+            out.append(ts)
+            ts += timedelta(minutes=1)
+    return out
+
+
+def write_calendar(
+    root: Path,
+    rows: list[dict],
+    *,
+    market_file: str = "nse",
+) -> Path:
+    """Seed reference calendar (used by tests; not a public MarketStore write API)."""
+    path = calendar_path(root, market=market_file)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    table = pa.Table.from_pylist(rows, schema=CALENDAR_SCHEMA)
+    pq.write_table(table, path)
+    return path

tt_data_store/candles.py

@@ -0,0 +1,523 @@
+"""Candle read/write pipeline (docs 04, 06, 10)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+from zoneinfo import ZoneInfo
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from tt_data_store.atomic import atomic_replace, temp_path_for
+from tt_data_store.gaps import serialize_gaps
+from tt_data_store.instruments import Instrument
+from tt_data_store.paths import relative_path_for_instrument, years_in_range
+from tt_data_store.parquet import empty_candles, file_sha256, read_candles, write_candles
+from tt_data_store.schema import SCHEMA_VERSION, TZ
+from tt_data_store.validation import (
+    VALIDATOR_VERSION,
+    ValidationError,
+    assess_quality,
+    dedupe_exact,
+    filter_to_session,
+    merge_tables,
+    normalize,
+    validate_hard,
+)
+
+if TYPE_CHECKING:
+    from tt_data_store.store import MarketStore
+
+IST = ZoneInfo(TZ)
+
+
+@dataclass
+class WriteResult:
+    instrument: str
+    rows_received: int
+    rows_inserted: int
+    rows_duplicate: int
+    rows_conflicted: int
+    rows_rejected: int
+    previous_rows: int
+    new_rows: int
+    previous_gaps: list[list[str]] | None
+    new_gaps: list[list[str]] | None
+    status: str
+    file_changed: bool
+    path: str
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _resolve_instrument(store: MarketStore, instrument: Instrument | str) -> Instrument:
+    if isinstance(instrument, Instrument):
+        return instrument
+    found = store.instruments.get(instrument)
+    if found is None:
+        raise KeyError(f"unknown instrument: {instrument}")
+    return found
+
+
+def _paths_for_table(instrument: Instrument, table: pa.Table) -> dict[str, pa.Table]:
+    """Split a candle table into relative_path → subtable."""
+    parsed = instrument.parsed
+    if parsed.instrument_type != "INDEX":
+        path = relative_path_for_instrument(parsed)
+        return {path: table}
+
+    if table.num_rows == 0:
+        return {}
+
+    timestamps = table.column("timestamp").to_pylist()
+    by_year: dict[int, list[int]] = {}
+    for i, ts in enumerate(timestamps):
+        year = ts.astimezone(IST).year
+        by_year.setdefault(year, []).append(i)
+
+    out: dict[str, pa.Table] = {}
+    for year, indices in by_year.items():
+        path = relative_path_for_instrument(parsed, year=year)
+        out[path] = table.take(pa.array(indices))
+    return out
+
+
+class CandlesAPI:
+    def __init__(self, store: MarketStore) -> None:
+        self._store = store
+
+    def read(
+        self,
+        instrument: Instrument | str,
+        start: str | datetime | None = None,
+        end: str | datetime | None = None,
+    ) -> pa.Table:
+        inst = _resolve_instrument(self._store, instrument)
+        parsed = inst.parsed
+        tables: list[pa.Table] = []
+
+        if parsed.instrument_type == "INDEX":
+            if start is None or end is None:
+                # Read all known files for this instrument
+                files = self._store.files.list(instrument_id=inst.id)
+                paths = [f.path for f in files]
+            else:
+                paths = [
+                    relative_path_for_instrument(parsed, year=y)
+                    for y in years_in_range(start, end)
+                ]
+        else:
+            paths = [relative_path_for_instrument(parsed)]
+
+        for rel in paths:
+            abs_path = self._store.root / rel
+            if abs_path.exists():
+                tables.append(read_candles(abs_path))
+
+        if not tables:
+            return empty_candles()
+
+        combined = pa.concat_tables(tables)
+        if start is not None:
+            start_ts = _parse_bound(start, end=False)
+            combined = combined.filter(
+                pc.greater_equal(combined.column("timestamp"), start_ts)
+            )
+        if end is not None:
+            end_ts = _parse_bound(end, end=True)
+            combined = combined.filter(
+                pc.less_equal(combined.column("timestamp"), end_ts)
+            )
+        if combined.num_rows == 0:
+            return empty_candles()
+        # Ensure sorted
+        indices = pc.sort_indices(combined, sort_keys=[("timestamp", "ascending")])
+        return combined.take(indices)
+
+    def write(
+        self,
+        instrument: Instrument | str,
+        candles: pa.Table | list[dict[str, Any]],
+        *,
+        source: str,
+        source_instrument_id: str | None = None,
+        collector_version: str | None = None,
+        requested_start: str | None = None,
+        requested_end: str | None = None,
+    ) -> WriteResult:
+        inst = _resolve_instrument(self._store, instrument)
+        started_at = _now()
+        ingestion_id = self._insert_ingestion(
+            source=source,
+            source_instrument_id=source_instrument_id,
+            collector_version=collector_version,
+            requested_start=requested_start,
+            requested_end=requested_end,
+            started_at=started_at,
+            status="RUNNING",
+        )
+
+        try:
+            if isinstance(candles, list):
+                table = pa.Table.from_pylist(candles)
+            else:
+                table = candles
+
+            rows_received = table.num_rows
+            table = normalize(table)
+            table, batch_dups = dedupe_exact(table)
+            validate_hard(table)
+
+            # Partition by target file
+            partitions = _paths_for_table(inst, table)
+            if not partitions:
+                self._finish_ingestion(
+                    ingestion_id,
+                    received=rows_received,
+                    accepted=0,
+                    rejected=0,
+                    status="COMPLETED",
+                )
+                path = relative_path_for_instrument(inst.parsed) if inst.parsed.instrument_type != "INDEX" else ""
+                return WriteResult(
+                    instrument=inst.instrument_key,
+                    rows_received=rows_received,
+                    rows_inserted=0,
+                    rows_duplicate=batch_dups,
+                    rows_conflicted=0,
+                    rows_rejected=0,
+                    previous_rows=0,
+                    new_rows=0,
+                    previous_gaps=None,
+                    new_gaps=None,
+                    status="UNKNOWN",
+                    file_changed=False,
+                    path=path,
+                )
+
+            # For multi-year INDEX writes, process each partition; return last/aggregate
+            total_inserted = 0
+            total_dup = batch_dups
+            total_rejected = 0
+            last_result: WriteResult | None = None
+
+            for rel_path, part in partitions.items():
+                result = self._write_partition(
+                    inst=inst,
+                    rel_path=rel_path,
+                    incoming=part,
+                    rows_received=part.num_rows,
+                    batch_dups=0,
+                )
+                total_inserted += result.rows_inserted
+                total_dup += result.rows_duplicate
+                total_rejected += result.rows_rejected
+                last_result = result
+
+            assert last_result is not None
+            self._finish_ingestion(
+                ingestion_id,
+                received=rows_received,
+                accepted=total_inserted,
+                rejected=total_rejected,
+                status="COMPLETED",
+            )
+            return WriteResult(
+                instrument=inst.instrument_key,
+                rows_received=rows_received,
+                rows_inserted=total_inserted,
+                rows_duplicate=total_dup,
+                rows_conflicted=0,
+                rows_rejected=total_rejected,
+                previous_rows=last_result.previous_rows,
+                new_rows=last_result.new_rows,
+                previous_gaps=last_result.previous_gaps,
+                new_gaps=last_result.new_gaps,
+                status=last_result.status,
+                file_changed=last_result.file_changed,
+                path=last_result.path,
+            )
+        except ValidationError:
+            self._finish_ingestion(
+                ingestion_id,
+                received=0,
+                accepted=0,
+                rejected=0,
+                status="FAILED",
+            )
+            raise
+        except Exception:
+            self._finish_ingestion(
+                ingestion_id,
+                received=0,
+                accepted=0,
+                rejected=0,
+                status="FAILED",
+            )
+            raise
+
+    def _write_partition(
+        self,
+        *,
+        inst: Instrument,
+        rel_path: str,
+        incoming: pa.Table,
+        rows_received: int,
+        batch_dups: int,
+    ) -> WriteResult:
+        abs_path = self._store.root / rel_path
+        existing = read_candles(abs_path) if abs_path.exists() else empty_candles()
+        previous_rows = existing.num_rows
+
+        prev_meta = self._store.files.get(rel_path)
+        previous_gaps = prev_meta.gaps if prev_meta else None
+
+        try:
+            merged, inserted, duplicates = merge_tables(existing, incoming)
+        except ValidationError:
+            # Conflict: existing untouched
+            raise
+
+        validate_hard(merged)
+
+        # Drop out-of-session rows from canonical file when calendar exists
+        filtered, rejected = filter_to_session(
+            self._store.root, merged, exchange=inst.exchange
+        )
+        if rejected:
+            merged = filtered
+            validate_hard(merged)
+
+        quality = assess_quality(self._store.root, merged, exchange=inst.exchange)
+
+        file_changed = inserted > 0 or not abs_path.exists() or rejected > 0
+        # Also change if first write of empty→empty? no
+        if inserted == 0 and duplicates == rows_received and abs_path.exists() and rejected == 0:
+            file_changed = False
+
+        if file_changed or not abs_path.exists():
+            tmp = temp_path_for(abs_path)
+            try:
+                write_candles(tmp, merged)
+                atomic_replace(tmp, abs_path)
+            finally:
+                if tmp.exists():
+                    tmp.unlink(missing_ok=True)
+
+            checksum = file_sha256(abs_path)
+            size_bytes = abs_path.stat().st_size
+            start_time = end_time = None
+            if merged.num_rows:
+                ts = merged.column("timestamp").to_pylist()
+                start_time = ts[0].astimezone(IST).isoformat()
+                end_time = ts[-1].astimezone(IST).isoformat()
+
+            self._upsert_file(
+                instrument_id=inst.id,
+                path=rel_path,
+                start_time=start_time,
+                end_time=end_time,
+                quality=quality,
+                size_bytes=size_bytes,
+                checksum=checksum,
+            )
+        elif prev_meta is None and abs_path.exists():
+            # Register existing file metadata after no-op merge on first catalogue miss
+            checksum = file_sha256(abs_path)
+            size_bytes = abs_path.stat().st_size
+            start_time = end_time = None
+            if merged.num_rows:
+                ts = merged.column("timestamp").to_pylist()
+                start_time = ts[0].astimezone(IST).isoformat()
+                end_time = ts[-1].astimezone(IST).isoformat()
+            self._upsert_file(
+                instrument_id=inst.id,
+                path=rel_path,
+                start_time=start_time,
+                end_time=end_time,
+                quality=quality,
+                size_bytes=size_bytes,
+                checksum=checksum,
+            )
+
+        return WriteResult(
+            instrument=inst.instrument_key,
+            rows_received=rows_received,
+            rows_inserted=inserted,
+            rows_duplicate=duplicates + batch_dups,
+            rows_conflicted=0,
+            rows_rejected=rejected,
+            previous_rows=previous_rows,
+            new_rows=merged.num_rows,
+            previous_gaps=previous_gaps,
+            new_gaps=quality.gaps,
+            status=quality.status,
+            file_changed=file_changed,
+            path=rel_path,
+        )
+
+    def _upsert_file(
+        self,
+        *,
+        instrument_id: int,
+        path: str,
+        start_time: str | None,
+        end_time: str | None,
+        quality: Any,
+        size_bytes: int,
+        checksum: str,
+    ) -> None:
+        now = _now()
+        gaps_raw = serialize_gaps(quality.gaps)
+        existing = self._store.files.get(path)
+        conn = self._store.conn
+        if existing is None:
+            cur = conn.execute(
+                """
+                INSERT INTO files (
+                    instrument_id, path, format, start_time, end_time,
+                    expected_rows, actual_rows, missing_rows, duplicate_rows,
+                    invalid_rows, gaps, size_bytes, checksum_sha256,
+                    schema_version, status, created_at, updated_at, validated_at
+                ) VALUES (?, ?, 'parquet', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    instrument_id,
+                    path,
+                    start_time,
+                    end_time,
+                    quality.expected_rows,
+                    quality.actual_rows,
+                    quality.missing_rows,
+                    quality.duplicate_rows,
+                    quality.invalid_rows,
+                    gaps_raw,
+                    size_bytes,
+                    checksum,
+                    SCHEMA_VERSION,
+                    quality.status,
+                    now,
+                    now,
+                    now,
+                ),
+            )
+            file_id = cur.lastrowid
+        else:
+            conn.execute(
+                """
+                UPDATE files SET
+                    start_time = ?, end_time = ?,
+                    expected_rows = ?, actual_rows = ?,
+                    missing_rows = ?, duplicate_rows = ?, invalid_rows = ?,
+                    gaps = ?, size_bytes = ?, checksum_sha256 = ?,
+                    schema_version = ?, status = ?,
+                    updated_at = ?, validated_at = ?
+                WHERE path = ?
+                """,
+                (
+                    start_time,
+                    end_time,
+                    quality.expected_rows,
+                    quality.actual_rows,
+                    quality.missing_rows,
+                    quality.duplicate_rows,
+                    quality.invalid_rows,
+                    gaps_raw,
+                    size_bytes,
+                    checksum,
+                    SCHEMA_VERSION,
+                    quality.status,
+                    now,
+                    now,
+                    path,
+                ),
+            )
+            file_id = existing.id
+
+        conn.execute(
+            """
+            INSERT INTO validation_runs (
+                file_id, validator_version, expected_rows, actual_rows,
+                missing_rows, duplicate_rows, invalid_rows, status,
+                started_at, completed_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                file_id,
+                VALIDATOR_VERSION,
+                quality.expected_rows,
+                quality.actual_rows,
+                quality.missing_rows,
+                quality.duplicate_rows,
+                quality.invalid_rows,
+                quality.status,
+                now,
+                now,
+            ),
+        )
+        conn.commit()
+
+    def _insert_ingestion(self, **kwargs: Any) -> int:
+        cur = self._store.conn.execute(
+            """
+            INSERT INTO ingestions (
+                source, source_instrument_id, requested_start, requested_end,
+                received_rows, accepted_rows, rejected_rows, collector_version,
+                started_at, completed_at, status
+            ) VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?, NULL, ?)
+            """,
+            (
+                kwargs["source"],
+                kwargs.get("source_instrument_id"),
+                kwargs.get("requested_start"),
+                kwargs.get("requested_end"),
+                kwargs.get("collector_version"),
+                kwargs["started_at"],
+                kwargs["status"],
+            ),
+        )
+        self._store.conn.commit()
+        return int(cur.lastrowid)
+
+    def _finish_ingestion(
+        self,
+        ingestion_id: int,
+        *,
+        received: int,
+        accepted: int,
+        rejected: int,
+        status: str,
+    ) -> None:
+        self._store.conn.execute(
+            """
+            UPDATE ingestions SET
+                received_rows = ?, accepted_rows = ?, rejected_rows = ?,
+                completed_at = ?, status = ?
+            WHERE id = ?
+            """,
+            (received, accepted, rejected, _now(), status, ingestion_id),
+        )
+        self._store.conn.commit()
+
+
+def _parse_bound(value: str | datetime, *, end: bool) -> datetime:
+    if isinstance(value, datetime):
+        ts = value
+    else:
+        text = value
+        if len(text) == 10:
+            # date only
+            if end:
+                text = text + "T23:59:00"
+            else:
+                text = text + "T00:00:00"
+        ts = datetime.fromisoformat(text)
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=IST)
+    else:
+        ts = ts.astimezone(IST)
+    return ts

tt_data_store/db.py

@@ -0,0 +1,40 @@
+"""SQLite metadata database for the archive catalogue."""
+
+from __future__ import annotations
+
+import sqlite3
+from importlib import resources
+from pathlib import Path
+
+
+def connect(db_path: Path) -> sqlite3.Connection:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA foreign_keys = ON")
+    return conn
+
+
+def init_schema(conn: sqlite3.Connection) -> None:
+    sql = resources.files("tt_data_store").joinpath("schema.sql").read_text(encoding="utf-8")
+    conn.executescript(sql)
+    conn.commit()
+
+
+def migrate_schema(conn: sqlite3.Connection) -> None:
+    """Apply additive migrations for existing archives."""
+    cols = {row[1] for row in conn.execute("PRAGMA table_info(files)")}
+    if "last_synced_checksum_sha256" not in cols:
+        conn.execute(
+            "ALTER TABLE files ADD COLUMN last_synced_checksum_sha256 TEXT"
+        )
+        conn.commit()
+
+
+def open_or_create(db_path: Path) -> sqlite3.Connection:
+    exists = db_path.exists()
+    conn = connect(db_path)
+    if not exists:
+        init_schema(conn)
+    else:
+        migrate_schema(conn)
+    return conn