onesecondtrader 0.52.0__py3-none-any.whl → 0.54.0__py3-none-any.whl

onesecondtrader/datafeeds/__init__.py

@@ -0,0 +1,11 @@
+"""
+Provides data feed components for ingesting market data into the system.
+"""
+
+from .base import DatafeedBase
+from .simulated import SimulatedDatafeed
+
+__all__ = [
+    "DatafeedBase",
+    "SimulatedDatafeed",
+]

onesecondtrader/datafeeds/base.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import abc
+
+from onesecondtrader import events, messaging, models
+
+
+class DatafeedBase(abc.ABC):
+    """
+    Abstract base class for market data feed implementations.
+
+    A data feed is responsible for connecting to an external data source, managing symbol and bar-period subscriptions, and publishing market data events onto the system event bus.
+
+    Concrete subclasses implement the mechanics of connectivity, subscription handling, and lifecycle management for a specific data source.
+    """
+
+    def __init__(self, event_bus: messaging.EventBus) -> None:
+        """
+        Initialize the data feed with an event bus.
+
+        parameters:
+            event_bus:
+                Event bus used to publish market data events produced by this data feed.
+        """
+        self._event_bus = event_bus
+
+    def _publish(self, event: events.EventBase) -> None:
+        """
+        Publish a market data event to the event bus.
+
+        This method is intended for use by subclasses to forward incoming data from the external source into the internal event-driven system.
+
+        parameters:
+            event:
+                Event instance to be published.
+        """
+        self._event_bus.publish(event)
+
+    @abc.abstractmethod
+    def connect(self) -> None:
+        """
+        Establish a connection to the underlying data source.
+
+        Implementations should perform any required setup, authentication, or resource allocation needed before subscriptions can be registered.
+        """
+        pass
+
+    @abc.abstractmethod
+    def disconnect(self) -> None:
+        """
+        Terminate the connection to the underlying data source.
+
+        Implementations should release resources and ensure that no further events are published after disconnection.
+        """
+        pass
+
+    @abc.abstractmethod
+    def subscribe(self, symbols: list[str], bar_period: models.BarPeriod) -> None:
+        """
+        Subscribe to market data for one or more symbols at a given bar period.
+
+        parameters:
+            symbols:
+                Instrument symbols to subscribe to, interpreted according to the conventions of the underlying data source.
+            bar_period:
+                Bar aggregation period specifying the granularity of market data.
+        """
+        pass
+
+    @abc.abstractmethod
+    def unsubscribe(self, symbols: list[str], bar_period: models.BarPeriod) -> None:
+        """
+        Cancel existing subscriptions for one or more symbols at a given bar period.
+
+        parameters:
+            symbols:
+                Instrument symbols for which subscriptions should be removed.
+            bar_period:
+                Bar aggregation period associated with the subscriptions.
+        """
+        pass
+
+    def wait_until_complete(self) -> None:
+        """
+        Block until the data feed has completed all pending work.
+
+        This method may be overridden by subclasses that perform asynchronous ingestion or background processing.
+        The default implementation does nothing.
+        """
+        pass

onesecondtrader/datafeeds/simulated.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import itertools
+import os
+import sqlite3
+import threading
+
+from onesecondtrader import events, messaging, models
+from onesecondtrader.datafeeds.base import DatafeedBase
+
+_RTYPE_MAP = {
+    models.BarPeriod.SECOND: 32,
+    models.BarPeriod.MINUTE: 33,
+    models.BarPeriod.HOUR: 34,
+    models.BarPeriod.DAY: 35,
+}
+
+_RTYPE_TO_BAR_PERIOD = {v: k for k, v in _RTYPE_MAP.items()}
+
+
+class SimulatedDatafeed(DatafeedBase):
+    """
+    Simulated market data feed backed by a secmaster SQLite database.
+
+    This datafeed replays historical OHLCV bars from a secmaster database, resolving symbols
+    via time-bounded symbology mappings. Bars are delivered in timestamp order, with all bars
+    sharing the same timestamp published before calling `wait_until_system_idle`.
+
+    Subclasses must set `publisher_name`, `dataset`, and `symbol_type` as class attributes to
+    scope the feed to a specific data source. The database must contain publishers with numeric
+    `source_instrument_id` values; symbol-only publishers (e.g., yfinance) are not supported.
+    """
+
+    db_path: str = ""
+    publisher_name: str = ""
+    dataset: str = ""
+    symbol_type: str = ""
+    price_scale: float = 1e9
+    start_ts: int | None = None
+    end_ts: int | None = None
+
+    def __init__(self, event_bus: messaging.EventBus) -> None:
+        """
+        Parameters:
+            event_bus:
+                Event bus used to publish bar events and synchronize with subscribers.
+        """
+        super().__init__(event_bus)
+        self._db_path = self.db_path or os.environ.get(
+            "SECMASTER_DB_PATH", "secmaster.db"
+        )
+        if not self.publisher_name:
+            raise ValueError("publisher_name is required")
+        if not self.dataset:
+            raise ValueError("dataset is required")
+        if not self.symbol_type:
+            raise ValueError("symbol_type is required")
+        self._subscriptions: set[tuple[str, models.BarPeriod]] = set()
+        self._subscriptions_lock = threading.Lock()
+        self._connection: sqlite3.Connection | None = None
+        self._thread: threading.Thread | None = None
+        self._stop_event = threading.Event()
+        self._publisher_id: int | None = None
+
+    def connect(self) -> None:
+        """
+        Open a connection to the secmaster database and resolve the publisher.
+
+        If already connected, this method returns immediately.
+        """
+        if self._connection:
+            return
+        self._connection = sqlite3.connect(self._db_path, check_same_thread=False)
+        self._connection.execute("PRAGMA foreign_keys = ON")
+        self._connection.execute("PRAGMA journal_mode = WAL")
+        row = self._connection.execute(
+            "SELECT publisher_id FROM publishers WHERE name = ? AND dataset = ?",
+            (self.publisher_name, self.dataset),
+        ).fetchone()
+        if row is None:
+            raise ValueError(
+                f"Publisher not found: {self.publisher_name}/{self.dataset}"
+            )
+        self._publisher_id = row[0]
+
+    def disconnect(self) -> None:
+        """
+        Close the database connection and stop any active streaming.
+
+        If not connected, this method returns immediately.
+        """
+        if not self._connection:
+            return
+        self._stop_event.set()
+        if self._thread and self._thread.is_alive():
+            self._thread.join()
+        self._connection.close()
+        self._connection = None
+        self._publisher_id = None
+
+    def subscribe(self, symbols: list[str], bar_period: models.BarPeriod) -> None:
+        """
+        Register symbols for bar delivery at the specified period.
+
+        Parameters:
+            symbols:
+                List of ticker symbols to subscribe.
+            bar_period:
+                Bar aggregation period for the subscription.
+        """
+        with self._subscriptions_lock:
+            self._subscriptions.update((s, bar_period) for s in symbols)
+
+    def unsubscribe(self, symbols: list[str], bar_period: models.BarPeriod) -> None:
+        """
+        Remove symbols from bar delivery at the specified period.
+
+        Parameters:
+            symbols:
+                List of ticker symbols to unsubscribe.
+            bar_period:
+                Bar aggregation period for the subscription.
+        """
+        with self._subscriptions_lock:
+            self._subscriptions.difference_update((s, bar_period) for s in symbols)
+
+    def wait_until_complete(self) -> None:
+        """
+        Stream all subscribed bars and block until delivery is complete.
+
+        Bars are published in timestamp order. After each timestamp batch, the method
+        waits for all event bus subscribers to become idle before proceeding.
+        """
+        with self._subscriptions_lock:
+            has_subscriptions = bool(self._subscriptions)
+        if not has_subscriptions:
+            return
+        if self._thread is None or not self._thread.is_alive():
+            self._stop_event.clear()
+            self._thread = threading.Thread(
+                target=self._stream,
+                name=self.__class__.__name__,
+                daemon=False,
+            )
+            self._thread.start()
+        self._thread.join()
+
+    def _stream(self) -> None:
+        if not self._connection or self._publisher_id is None:
+            return
+
+        with self._subscriptions_lock:
+            subscriptions = list(self._subscriptions)
+        if not subscriptions:
+            return
+
+        symbols = list({symbol for symbol, _ in subscriptions})
+        rtypes = list({_RTYPE_MAP[bp] for _, bp in subscriptions})
+        subscription_set = {(symbol, _RTYPE_MAP[bp]) for symbol, bp in subscriptions}
+
+        params: list = [self._publisher_id, self.symbol_type]
+        params.extend(symbols)
+        params.extend(rtypes)
+        if self.start_ts is not None:
+            params.append(self.start_ts)
+        if self.end_ts is not None:
+            params.append(self.end_ts)
+
+        query = f"""
+            SELECT s.symbol, o.rtype, o.ts_event, o.open, o.high, o.low, o.close, o.volume
+            FROM ohlcv o
+            JOIN instruments i ON i.instrument_id = o.instrument_id
+            JOIN symbology s
+              ON s.publisher_ref = i.publisher_ref
+             AND s.source_instrument_id = i.source_instrument_id
+             AND date(o.ts_event / 1000000000, 'unixepoch') >= s.start_date
+             AND date(o.ts_event / 1000000000, 'unixepoch') <= s.end_date
+            WHERE i.publisher_ref = ?
+              AND s.symbol_type = ?
+              AND s.symbol IN ({",".join("?" * len(symbols))})
+              AND o.rtype IN ({",".join("?" * len(rtypes))})
+              {"AND o.ts_event >= ?" if self.start_ts is not None else ""}
+              {"AND o.ts_event <= ?" if self.end_ts is not None else ""}
+            ORDER BY o.ts_event, s.symbol
+        """
+
+        rows = self._connection.execute(query, params)
+
+        def to_bar(row):
+            symbol, rtype, ts_event, open_, high, low, close, volume = row
+            if (symbol, rtype) not in subscription_set:
+                return None
+            return events.market.BarReceived(
+                ts_event_ns=ts_event,
+                symbol=symbol,
+                bar_period=_RTYPE_TO_BAR_PERIOD[rtype],
+                open=open_ / self.price_scale,
+                high=high / self.price_scale,
+                low=low / self.price_scale,
+                close=close / self.price_scale,
+                volume=volume,
+            )
+
+        for _, group in itertools.groupby(rows, key=lambda r: r[2]):
+            if self._stop_event.is_set():
+                return
+            for bar in filter(None, map(to_bar, group)):
+                self._publish(bar)
+            self._event_bus.wait_until_system_idle()

onesecondtrader/secmaster/__init__.py

@@ -1,3 +1,15 @@
 """
 Provides a schema for creating and utilities to populate the security master database.
 """
+
+from .utils import (
+    create_secmaster_db,
+    ingest_databento_zip,
+    ingest_databento_dbn,
+)
+
+__all__ = [
+    "create_secmaster_db",
+    "ingest_databento_zip",
+    "ingest_databento_dbn",
+]

onesecondtrader/secmaster/schema_versions/secmaster_schema_v1.sql

@@ -4,10 +4,16 @@
 -- Instrument identity is modeled per publisher namespace and supports either numeric upstream identifiers or symbols.
 -- Contract specifications and other static reference metadata are intentionally out of scope for this schema and should be stored separately if ingested.
 --
+-- The schema is explicitly ingestion-safe in the sense that:
+--
+-- 1) publishers are keyed by (vendor, dataset) rather than vendor alone, allowing multiple feeds per vendor;
+-- 2) symbology admits multiple mappings sharing the same start date by including the resolved instrument identifier
+--    in the primary key, preventing accidental overwrites during bulk ingestion.
+--
 -- | Table         | Description |
 -- |---------------|-------------|
--- | `publishers`  | Registry of data sources and their identifier namespaces. |
--- | `instruments` | Registry of instruments observed from market data ingestion within a publisher namespace. |
+-- | `publishers`  | Registry of vendor+dataset namespaces used for market data and instrument ingestion. |
+-- | `instruments` | Registry of instruments observed from ingestion within a publisher namespace. |
 -- | `ohlcv`       | Aggregated OHLCV bar data keyed by instrument, bar duration (`rtype`), and event timestamp (`ts_event`). |
 -- | `symbology`   | Time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers. |
 
@@ -15,15 +21,24 @@
 
 -- Registry of all data sources used for market data and instrument ingestion.
 --
--- Each row represents a distinct data source.
--- A publisher establishes the provenance of instrument definitions and price data and provides the context in which raw symbols and native instrument identifiers are interpreted.
+-- Each row represents a distinct data product (feed) within a vendor namespace.
+-- A publisher record is uniquely identified by the pair (`name`, `dataset`), not by `name` alone.
+-- This allows a single vendor (e.g. Databento) to appear multiple times, once per concrete dataset/feed
+-- (e.g. `GLBX.MDP3`, `XNAS.ITCH`).
+--
+-- A publisher establishes the provenance of instrument definitions and price data and provides the context
+-- in which raw symbols and native instrument identifiers are interpreted.
 --
--- | Field           | Type      | Constraints          | Description                                                                                                                                                                     |
--- |-----------------|-----------|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
--- | `publisher_id`  | `INTEGER` | `PRIMARY KEY`        | Internal surrogate key uniquely identifying a data source within the system.                                                                                                    |
--- | `name`          | `TEXT`    | `NOT NULL`, `UNIQUE` | Human-readable identifier for the data source or vendor (e.g. `databento`, `yfinance`).                                                                                         |
--- | `dataset`       | `TEXT`    | `NOT NULL`           | Identifier of the concrete data product or feed through which data is sourced; uses Databento dataset names (e.g. `GLBX.MDP3`) for Databento ingestion and internal identifiers for other sources (e.g. `YFINANCE`). |
--- | `venue`         | `TEXT`    |                      | Optional ISO 10383 Market Identifier Code (MIC) describing the primary trading venue; may be NULL for aggregated or multi-venue sources.                                        |
+-- | Field           | Type      | Constraints             | Description                                                                                                                                                                     |
+-- |-----------------|-----------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `publisher_id`  | `INTEGER` | `PRIMARY KEY`           | Internal surrogate key uniquely identifying a publisher record within the system.                                                                                               |
+-- | `name`          | `TEXT`    | `NOT NULL`              | Human-readable vendor identifier for the data source (e.g. `databento`, `yfinance`).                                                                                            |
+-- | `dataset`       | `TEXT`    | `NOT NULL`              | Identifier of the concrete data product or feed through which data is sourced; uses Databento dataset names (e.g. `GLBX.MDP3`) for Databento ingestion and internal identifiers for other sources (e.g. `YFINANCE`). |
+-- | `venue`         | `TEXT`    |                         | Optional ISO 10383 Market Identifier Code (MIC) describing the primary trading venue; may be NULL for aggregated or multi-venue sources.                                        |
+--
+-- **Table constraints**
+--
+-- * `UNIQUE(name, dataset)` ensures that each vendor+feed combination is represented at most once.
 --
 -- **Examples**
 --
@@ -33,6 +48,12 @@
 -- *  `dataset` = `'GLBX.MDP3'`
 -- *  `venue`   = `XCME`
 --
+-- Databento NASDAQ TotalView feed:
+--
+-- *  `name`    = `'databento'`
+-- *  `dataset` = `'XNAS.ITCH'`
+-- *  `venue`   = `XNAS`
+--
 -- Yahoo Finance equity data:
 --
 -- *  `name`    = `'yfinance'`
@@ -41,14 +62,16 @@
 --
 CREATE TABLE publishers (
     publisher_id INTEGER PRIMARY KEY,
-    name TEXT NOT NULL UNIQUE,
+    name TEXT NOT NULL,
     dataset TEXT NOT NULL,
-    venue TEXT
+    venue TEXT,
+    UNIQUE (name, dataset)
 );
 
 
 
 
+
 -- Registry of instruments observed through market data ingestion.
 --
 -- Each row represents an instrument identity within a publisher namespace.
@@ -59,13 +82,13 @@ CREATE TABLE publishers (
 -- The table does not store contract specifications or other reference metadata.
 -- Such metadata must be stored separately when available.
 --
--- | Field                  | Type      | Constraints      | Description                                                                                                                                                 |
--- |------------------------|-----------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
--- | `instrument_id`        | `INTEGER` | `PRIMARY KEY`    | Internal surrogate key identifying an instrument record within the system.                                                                                  |
--- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which this instrument identity is valid.                            |
+-- | Field                  | Type      | Constraints      | Description                                                                                                                                                  |
+-- |------------------------|-----------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id`        | `INTEGER` | `PRIMARY KEY`    | Internal surrogate key identifying an instrument record within the system.                                                                                   |
+-- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which this instrument identity is valid.                             |
 -- | `source_instrument_id` | `INTEGER` |                  | Publisher-native numeric instrument identifier as provided by the upstream data source (e.g. Databento instrument_id); may be `NULL` for symbol-only sources. |
--- | `symbol`               | `TEXT`    |                  | Publisher-native symbol string identifying the instrument (e.g. raw symbol, ticker); may be NULL when numeric identifiers are used.                         |
--- | `symbol_type`          | `TEXT`    |                  | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`, `ticker`).                               |
+-- | `symbol`               | `TEXT`    |                  | Publisher-native symbol string identifying the instrument (e.g. raw symbol, ticker); may be NULL when numeric identifiers are used.                          |
+-- | `symbol_type`          | `TEXT`    |                  | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`, `ticker`).                                |
 --
 -- Each instrument must be identifiable by at least one of `source_instrument_id` or `symbol`.
 -- Uniqueness constraints ensure that instrument identities do not collide within a publisher namespace.
@@ -78,7 +101,7 @@ CREATE TABLE instruments (
 
     source_instrument_id INTEGER,
     symbol TEXT,
-    symbol_type TEXT,
+	symbol_type TEXT,
 
     FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
 
@@ -87,6 +110,8 @@ CREATE TABLE instruments (
         OR symbol IS NOT NULL
     ),
 
+	CHECK (symbol IS NULL OR symbol_type IS NOT NULL),
+
     UNIQUE (publisher_ref, source_instrument_id),
     UNIQUE (publisher_ref, symbol, symbol_type)
 );
@@ -98,16 +123,16 @@ CREATE TABLE instruments (
 
 -- Stores aggregated OHLCV bars for instruments at multiple time resolutions.
 --
--- | Field           | Type      | Constraints                                 | Description                                                                                                             |
--- |-----------------|-----------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
--- | `instrument_id` | `INTEGER` | `NOT NULL`, `FK`                            | Foreign key reference to `instruments.instrument_id`, identifying the instrument to which this bar belongs.             |
+-- | Field           | Type      | Constraints                                 | Description                                                                                                              |
+-- |-----------------|-----------|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id` | `INTEGER` | `NOT NULL`, `FK`                            | Foreign key reference to `instruments.instrument_id`, identifying the instrument to which this bar belongs.              |
 -- | `rtype`         | `INTEGER` | `NOT NULL`, `CHECK IN (32, 33, 34, 35, 36)` | Record type code encoding the bar duration using Databento OHLCV conventions (e.g. `32`=1s, `33`=1m, `34`=1h, `35`=1d). |
--- | `ts_event`      | `INTEGER` | `NOT NULL`                                  | Event timestamp of the bar as provided by the upstream source, stored as nanoseconds since the UTC Unix epoch.              |
--- | `open`          | `INTEGER` | `NOT NULL`                                  | Opening price of the bar interval, stored as a fixed-point integer using the upstream price scaling convention.         |
--- | `high`          | `INTEGER` | `NOT NULL`                                  | Highest traded price during the bar interval, stored as a fixed-point integer.                                          |
--- | `low`           | `INTEGER` | `NOT NULL`, `CHECK(low <= high)`            | Lowest traded price during the bar interval, stored as a fixed-point integer.                                           |
--- | `close`         | `INTEGER` | `NOT NULL`                                  | Closing price of the bar interval, stored as a fixed-point integer.                                                     |
--- | `volume`        | `INTEGER` | `NOT NULL`, `CHECK(volume >= 0)`            | Total traded volume during the bar interval.                                                                            |
+-- | `ts_event`      | `INTEGER` | `NOT NULL`                                  | Event timestamp of the bar as provided by the upstream source, stored as nanoseconds since the UTC Unix epoch.           |
+-- | `open`          | `INTEGER` | `NOT NULL`                                  | Opening price of the bar interval, stored as a fixed-point integer using the upstream price scaling convention.          |
+-- | `high`          | `INTEGER` | `NOT NULL`                                  | Highest traded price during the bar interval, stored as a fixed-point integer.                                           |
+-- | `low`           | `INTEGER` | `NOT NULL`, `CHECK(low <= high)`            | Lowest traded price during the bar interval, stored as a fixed-point integer.                                            |
+-- | `close`         | `INTEGER` | `NOT NULL`                                  | Closing price of the bar interval, stored as a fixed-point integer.                                                      |
+-- | `volume`        | `INTEGER` | `NOT NULL`, `CHECK(volume >= 0)`            | Total traded volume during the bar interval.                                                                             |
 --
 -- The composite primary key enforces uniqueness per instrument, bar duration, and event timestamp.
 -- Integrity constraints ensure basic OHLC consistency and prevent invalid price relationships from being stored.
@@ -135,7 +160,12 @@ CREATE TABLE ohlcv (
 
 -- Stores time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers.
 --
--- The table captures symbol resolution rules as provided by upstream data sources and must be interpreted within the namespace of a specific publisher.
+-- The table captures symbol resolution rules as provided by upstream data sources and must be interpreted within the
+-- namespace of a specific publisher.
+--
+-- The schema permits multiple mappings to share the same `start_date` for a given (`publisher_ref`, `symbol`, `symbol_type`)
+-- by including `source_instrument_id` in the primary key. This prevents accidental overwrite when upstream symbology exports
+-- contain same-day corrections, backfills, or parallel resolution segments.
 --
 -- | Field                  | Type      | Constraints      | Description                                                                                                                |
 -- |------------------------|-----------|------------------|----------------------------------------------------------------------------------------------------------------------------|
@@ -146,7 +176,7 @@ CREATE TABLE ohlcv (
 -- | `start_date`           | `TEXT`    | `NOT NULL`       | First calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.          |
 -- | `end_date`             | `TEXT`    | `NOT NULL`       | Last calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.           |
 --
--- The primary key enforces uniqueness of symbol mappings per publisher, symbol type, and start date.
+-- The primary key enforces uniqueness of mappings at the granularity of a resolved instrument.
 -- Date bounds are interpreted as closed intervals.
 --
 CREATE TABLE symbology (
@@ -157,8 +187,9 @@ CREATE TABLE symbology (
     start_date TEXT NOT NULL,
     end_date TEXT NOT NULL,
     FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
-    PRIMARY KEY (publisher_ref, symbol, symbol_type, start_date),
-    UNIQUE (publisher_ref, symbol_type, source_instrument_id, start_date),
+    FOREIGN KEY (publisher_ref, source_instrument_id)
+        REFERENCES instruments(publisher_ref, source_instrument_id),
+    PRIMARY KEY (publisher_ref, symbol, symbol_type, start_date, source_instrument_id),
     CHECK (start_date <= end_date)
 );
 

onesecondtrader/secmaster/utils.py

@@ -1,16 +1,32 @@
 from __future__ import annotations
 
+import json
+import logging
 import pathlib
+import shutil
 import sqlite3
+import tempfile
+import zipfile
+
+import databento
+
+
+BATCH_SIZE = 10000
+LOG_EVERY_OHLCV = 1_000_000
+LOG_EVERY_SYMBOLOGY = 50_000
+
+logger = logging.getLogger(__name__)
 
 
 def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathlib.Path:
     """
     Create a new security master SQLite database using a selected schema version.
 
-    The database file is created at the given path and initialized by executing the SQL script located in the `schema_versions` directory adjacent to this module.
+    The database file is created at the given path and initialized by executing the SQL script
+    located in the `schema_versions` directory adjacent to this module.
 
-    The function expects the schema script to set `PRAGMA user_version` to the corresponding schema version and verifies this after execution.
+    The function expects the schema script to set `PRAGMA user_version` to the corresponding
+    schema version and verifies this after execution.
 
     Parameters:
         db_path:
@@ -27,8 +43,8 @@ def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathl
         FileNotFoundError:
             If the schema script for `schema_version` does not exist.
         sqlite3.DatabaseError:
-            If the applied schema does not set the expected `user_version`
-            or if SQLite fails while executing the schema.
+            If the applied schema does not set the expected `user_version` or if SQLite fails
+            while executing the schema.
     """
     if db_path.exists():
         raise FileExistsError(f"Database already exists: {db_path}")
@@ -57,8 +73,539 @@ def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathl
 
         if actual_version != schema_version:
             raise sqlite3.DatabaseError(
-                f"Schema script set user_version={actual_version}, "
-                f"expected {schema_version}"
+                f"Schema script set user_version={actual_version}, expected {schema_version}"
             )
 
     return db_path
+
+
+def ingest_databento_zip(
+    zip_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+    symbol_type: str = "raw_symbol",
+    dataset: str | None = None,
+) -> tuple[int, int]:
+    """
+    Ingest market data from a Databento zip archive into the security master database.
+
+    The archive may contain one or more DBN files and an optional `symbology.json`. The function
+    ingests OHLCV records from DBN files into `ohlcv` and ingests symbol-to-instrument mappings
+    into `symbology`.
+
+    The publisher namespace is created if absent. Publisher identity is determined by the pair
+    `(publisher_name, dataset)`, where `dataset` is extracted from `metadata.json` in the archive.
+
+    Ingestion is idempotent with respect to primary keys: existing `ohlcv` and `symbology` rows are
+    left unchanged.
+
+    Parameters:
+        zip_path:
+            Path to the Databento zip archive.
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from archive metadata.
+        symbol_type:
+            Symbol scheme stored in `symbology.symbol_type` for symbols found in `symbology.json`.
+        dataset:
+            Optional dataset override. If provided, it is used when `metadata.json` is missing or
+            does not specify a dataset.
+
+    Returns:
+        A tuple of (ohlcv_record_count_seen, symbology_record_count_seen).
+    """
+    ohlcv_count = 0
+    symbology_count = 0
+
+    logger.info("Opening Databento archive: %s", zip_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            with zipfile.ZipFile(zip_path, "r") as zf:
+                dataset, venue = _extract_dataset_info(zf, dataset_override=dataset)
+                logger.info(
+                    "Publisher resolved: name=%s dataset=%s venue=%s",
+                    publisher_name,
+                    dataset,
+                    venue,
+                )
+                publisher_id = _get_or_create_publisher(
+                    con, publisher_name, dataset, venue
+                )
+
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    dbn_files = [
+                        n
+                        for n in zf.namelist()
+                        if n.endswith(".dbn.zst") or n.endswith(".dbn")
+                    ]
+                    symbology_member = _zip_find_member(zf, "symbology.json")
+
+                    if not dbn_files and symbology_member is None:
+                        raise ValueError(
+                            "Archive contains no DBN files and no symbology.json"
+                        )
+
+                    logger.info("Found %d DBN file(s) in archive", len(dbn_files))
+
+                    for name in dbn_files:
+                        extracted_path = _zip_member_to_tempfile(zf, name, tmpdir)
+                        try:
+                            logger.info("Ingesting DBN file: %s", extracted_path.name)
+                            ohlcv_count += _ingest_dbn(
+                                extracted_path, con, publisher_id
+                            )
+                        finally:
+                            try:
+                                extracted_path.unlink()
+                            except FileNotFoundError:
+                                pass
+
+                    if symbology_member is not None:
+                        symbology_path = _zip_member_to_tempfile(
+                            zf, symbology_member, tmpdir
+                        )
+                        try:
+                            logger.info("Ingesting symbology.json")
+                            symbology_count += _ingest_symbology(
+                                symbology_path,
+                                con,
+                                publisher_id,
+                                symbol_type=symbol_type,
+                            )
+                        finally:
+                            try:
+                                symbology_path.unlink()
+                            except FileNotFoundError:
+                                pass
+                    else:
+                        logger.info("No symbology.json present in archive")
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info(
+        "Finished zip ingestion: %s (%d OHLCV records, %d symbology records)",
+        zip_path.name,
+        ohlcv_count,
+        symbology_count,
+    )
+
+    return ohlcv_count, symbology_count
+
+
+def ingest_databento_dbn(
+    dbn_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+) -> int:
+    """
+    Ingest market data from a Databento DBN file into the security master database.
+
+    Reads OHLCV records from the DBN file and inserts them into `ohlcv`. The publisher namespace
+    is created if absent. Publisher identity is determined by the pair `(publisher_name, dataset)`,
+    where `dataset` is read from DBN metadata.
+
+    Ingestion is idempotent with respect to primary keys: existing bars are left unchanged.
+
+    Parameters:
+        dbn_path:
+            Path to the DBN file (.dbn or .dbn.zst).
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from DBN metadata.
+
+    Returns:
+        The number of OHLCV records seen in the DBN stream.
+    """
+    logger.info("Starting DBN ingestion: %s", dbn_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            store = databento.DBNStore.from_file(dbn_path)
+            dataset = store.metadata.dataset
+            if not dataset:
+                raise ValueError(f"DBN metadata missing dataset: {dbn_path}")
+            venue = dataset.split(".")[0] if "." in dataset else None
+
+            logger.info(
+                "Publisher resolved: name=%s dataset=%s venue=%s",
+                publisher_name,
+                dataset,
+                venue,
+            )
+
+            publisher_id = _get_or_create_publisher(con, publisher_name, dataset, venue)
+            count = _ingest_dbn(dbn_path, con, publisher_id)
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info("Finished DBN ingestion: %s (%d OHLCV records)", dbn_path.name, count)
+
+    return count
+
+
+def _extract_dataset_info(
+    zf: zipfile.ZipFile,
+    dataset_override: str | None = None,
+) -> tuple[str, str | None]:
+    metadata_member = _zip_find_member(zf, "metadata.json")
+    if metadata_member is None:
+        if dataset_override is None:
+            raise ValueError(
+                "Archive is missing metadata.json and no dataset override was provided"
+            )
+        dataset = dataset_override
+    else:
+        with zf.open(metadata_member) as f:
+            metadata = json.load(f)
+        dataset = metadata.get("query", {}).get("dataset")
+        if not dataset:
+            if dataset_override is None:
+                raise ValueError(
+                    f"metadata.json is missing query.dataset (member={metadata_member!r})"
+                )
+            dataset = dataset_override
+
+    venue = dataset.split(".")[0] if "." in dataset else None
+    return dataset, venue
+
+
+def _zip_find_member(
+    zf: zipfile.ZipFile,
+    basename: str,
+    allow_multiple: bool = False,
+) -> str | None:
+    candidates = [
+        name
+        for name in zf.namelist()
+        if name == basename or name.endswith("/" + basename)
+    ]
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        return candidates[0]
+
+    candidates = sorted(candidates)
+    if not allow_multiple:
+        raise ValueError(f"Multiple {basename} members found in archive: {candidates}")
+
+    selected = candidates[0]
+    logger.warning("Multiple %s found in archive; using %s", basename, selected)
+    return selected
+
+
+def _zip_member_to_tempfile(
+    zf: zipfile.ZipFile,
+    member_name: str,
+    tmpdir: str,
+) -> pathlib.Path:
+    suffix = "".join(pathlib.PurePosixPath(member_name).suffixes)
+    with tempfile.NamedTemporaryFile(
+        mode="wb",
+        suffix=suffix,
+        delete=False,
+        dir=tmpdir,
+    ) as tmp:
+        with zf.open(member_name) as src:
+            shutil.copyfileobj(src, tmp)
+        return pathlib.Path(tmp.name)
+
+
+def _get_or_create_publisher(
+    con: sqlite3.Connection,
+    name: str,
+    dataset: str,
+    venue: str | None,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT publisher_id FROM publishers WHERE name = ? AND dataset = ?",
+        (name, dataset),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO publishers (name, dataset, venue) VALUES (?, ?, ?)",
+        (name, dataset, venue),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _get_or_create_instrument(
+    con: sqlite3.Connection,
+    publisher_id: int,
+    source_instrument_id: int,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT instrument_id FROM instruments WHERE publisher_ref = ? AND source_instrument_id = ?",
+        (publisher_id, source_instrument_id),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO instruments (publisher_ref, source_instrument_id) VALUES (?, ?)",
+        (publisher_id, source_instrument_id),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _assert_secmaster_db(
+    con: sqlite3.Connection, expected_user_version: int = 1
+) -> None:
+    row = con.execute("PRAGMA user_version;").fetchone()
+    user_version = int(row[0]) if row else 0
+    if user_version != expected_user_version:
+        raise sqlite3.DatabaseError(
+            "Security master schema user_version="
+            f"{user_version} does not match expected {expected_user_version}"
+        )
+
+    required = {"publishers", "instruments", "ohlcv", "symbology"}
+    present = {
+        r[0]
+        for r in con.execute(
+            "SELECT name FROM sqlite_master WHERE type = 'table'"
+        ).fetchall()
+    }
+    missing = sorted(required - present)
+    if missing:
+        raise sqlite3.DatabaseError(
+            f"Security master schema missing required tables: {', '.join(missing)}"
+        )
+
+
+def _ingest_dbn(
+    dbn_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+) -> int:
+    store = databento.DBNStore.from_file(dbn_path)
+    cursor = con.cursor()
+
+    instrument_cache: dict[int, int] = {}
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming OHLCV records from: %s", dbn_path.name)
+
+    for record in store:
+        if not isinstance(record, databento.OHLCVMsg):
+            continue
+
+        source_id = record.instrument_id
+        if source_id not in instrument_cache:
+            instrument_cache[source_id] = _get_or_create_instrument(
+                con, publisher_id, source_id
+            )
+        internal_id = instrument_cache[source_id]
+
+        rtype_val = (
+            record.rtype.value if hasattr(record.rtype, "value") else record.rtype
+        )
+
+        batch.append(
+            (
+                internal_id,
+                rtype_val,
+                record.ts_event,
+                record.open,
+                record.high,
+                record.low,
+                record.close,
+                record.volume,
+            )
+        )
+        count += 1
+
+        if count % LOG_EVERY_OHLCV == 0:
+            logger.info("Ingested %d OHLCV records from %s", count, dbn_path.name)
+
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                "INSERT OR IGNORE INTO ohlcv "
+                "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                batch,
+            )
+            batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO ohlcv "
+            "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    logger.info("Completed OHLCV ingest from %s (%d records)", dbn_path.name, count)
+
+    return count
+
+
+def _ingest_symbology(
+    json_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+    symbol_type: str = "raw_symbol",
+) -> int:
+    if not isinstance(symbol_type, str) or not symbol_type:
+        raise ValueError("symbol_type must be a non-empty string")
+
+    with open(json_path, "r") as f:
+        data = json.load(f)
+
+    if not isinstance(data, dict):
+        raise ValueError("symbology.json root must be a JSON object")
+
+    result = data.get("result", {})
+    if not isinstance(result, dict):
+        raise ValueError("symbology.json['result'] must be an object")
+    cursor = con.cursor()
+
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming symbology mappings from: %s", json_path.name)
+
+    instrument_cache: set[int] = set()
+
+    for symbol, mappings in result.items():
+        if not isinstance(mappings, list):
+            raise ValueError(
+                f"symbology.json mappings must be a list for symbol={symbol!r}"
+            )
+
+        for i, mapping in enumerate(mappings):
+            if not isinstance(mapping, dict):
+                raise ValueError(
+                    f"symbology.json mapping must be an object at symbol={symbol!r} index={i}"
+                )
+
+            missing_keys = [k for k in ("s", "d0", "d1") if k not in mapping]
+            if missing_keys:
+                raise ValueError(
+                    "symbology.json mapping missing key(s) "
+                    f"{missing_keys} at symbol={symbol!r} index={i}"
+                )
+
+            source_id = int(mapping["s"])
+
+            if source_id not in instrument_cache:
+                _get_or_create_instrument(con, publisher_id, source_id)
+                instrument_cache.add(source_id)
+
+            batch.append(
+                (
+                    publisher_id,
+                    symbol,
+                    symbol_type,
+                    source_id,
+                    mapping["d0"],
+                    mapping["d1"],
+                )
+            )
+            count += 1
+
+            if count % LOG_EVERY_SYMBOLOGY == 0:
+                logger.info(
+                    "Ingested %d symbology mappings from %s", count, json_path.name
+                )
+
+            if len(batch) >= BATCH_SIZE:
+                cursor.executemany(
+                    "INSERT OR IGNORE INTO symbology "
+                    "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+                    "VALUES (?, ?, ?, ?, ?, ?)",
+                    batch,
+                )
+                batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO symbology "
+            "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    _validate_no_overlapping_symbology(con, publisher_id, symbol_type)
+
+    logger.info(
+        "Completed symbology ingest from %s (%d mappings)", json_path.name, count
+    )
+
+    return count
+
+
+def _validate_no_overlapping_symbology(
+    con: sqlite3.Connection,
+    publisher_id: int,
+    symbol_type: str,
+) -> None:
+    query = """
+        WITH ordered AS (
+            SELECT
+                symbol,
+                start_date,
+                end_date,
+                LEAD(start_date) OVER (
+                    PARTITION BY symbol ORDER BY start_date
+                ) AS next_start
+            FROM symbology
+            WHERE publisher_ref = ? AND symbol_type = ?
+        )
+        SELECT symbol, start_date, end_date, next_start
+        FROM ordered
+        WHERE next_start IS NOT NULL AND end_date > next_start
+        LIMIT 1
+    """
+    row = con.execute(query, (publisher_id, symbol_type)).fetchone()
+    if row:
+        symbol, start, end, next_start = row
+        raise ValueError(
+            f"Overlapping symbology detected for symbol={symbol!r}: "
+            f"segment [{start}, {end}] overlaps with next segment starting {next_start}"
+        )
+
+
+def _enable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA journal_mode = WAL")
+    con.execute("PRAGMA synchronous = NORMAL")
+    con.execute("PRAGMA cache_size = -64000")
+
+
+def _disable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA synchronous = FULL")
+    con.execute("PRAGMA journal_mode = DELETE")
+    con.execute("PRAGMA cache_size = -2000")

{onesecondtrader-0.52.0.dist-info → onesecondtrader-0.54.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: onesecondtrader
-Version: 0.52.0
+Version: 0.54.0
 Summary: The Trading Infrastructure Toolkit for Python. Research, simulate, and deploy algorithmic trading strategies — all in one place.
 License-File: LICENSE
 Author: Nils P. Kujath

{onesecondtrader-0.52.0.dist-info → onesecondtrader-0.54.0.dist-info}/RECORD

@@ -2,6 +2,9 @@ onesecondtrader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onesecondtrader/brokers/__init__.py,sha256=CmOhwKOayuYCeg5KRiTp4fc8nSDnsLzIBkUNWhUevlo,271
 onesecondtrader/brokers/base.py,sha256=I4tQFr7P1DF5QAWb3I9tHz5D_zTleH8vEXS2WsH55DE,3531
 onesecondtrader/brokers/simulated.py,sha256=ZY39a84J2BmC2ADMkrSRzNBumPXudVBz2eUSnnHb0LM,17930
+onesecondtrader/datafeeds/__init__.py,sha256=uu67phyj4ruIlMqrsbFQ_mP5u2we881WgQoOhuN_KvU,214
+onesecondtrader/datafeeds/base.py,sha256=MOSUCuVfzPpqGf_T7O2aW64m9H2J4oa5u3VClQwZASE,3089
+onesecondtrader/datafeeds/simulated.py,sha256=LjPJU6RTt7i9e0pE0YSrtjmGLAQOGmmIpWSQMS9Dh6g,7840
 onesecondtrader/events/__init__.py,sha256=1T7hJA6afxClEXvvnbXtHu9iMyhduRdJZWlg4ObWaKE,222
 onesecondtrader/events/base.py,sha256=WpLo1bSKJe7Poh2IuDDCiBYZo9vE8mkq3cQlUpyTXsY,850
 onesecondtrader/events/market/__init__.py,sha256=49z6maexBIDkAjIfkLbYzSZWEbyTpQ_HEEgT0eacrDo,132
@@ -34,11 +37,11 @@ onesecondtrader/models/bar_period.py,sha256=J8ncVtcAxR52uD0nbC8Knds_GUP5wiuNj5rA
 onesecondtrader/models/order_types.py,sha256=SiJamarLQ7zkHzHLLbd86I_TeZrQJ4QEIMqNHj4dxXU,737
 onesecondtrader/models/rejection_reasons.py,sha256=Avp1JYf413_aUQQkEeswI-9EJBmQdd7B6bnQ1MslDNE,2132
 onesecondtrader/models/trade_sides.py,sha256=Pf9BpxoUxqgKC_EKAExfSqgfIIK9NW-RpJES0XHRF-8,583
-onesecondtrader/secmaster/__init__.py,sha256=miFKJ50zz3STh2HktAjYX8fPD8mzgWipr2EeiZfKdoM,95
+onesecondtrader/secmaster/__init__.py,sha256=XAouFrbRTpWWp8U43LQUkj8EZvJR9ydlI9fVdJjH1BY,294
 onesecondtrader/secmaster/schema_versions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-onesecondtrader/secmaster/schema_versions/secmaster_schema_v1.sql,sha256=-IeHE-nO0j8hJfgvaN07_0o4zoZ2oCsvSTncCUbrfvk,11540
-onesecondtrader/secmaster/utils.py,sha256=kh7djz1o8Yxl2lSMqu1oqWYs7l8_FNMaDb_8yvItIbQ,2172
-onesecondtrader-0.52.0.dist-info/METADATA,sha256=gKzl22R5cIZ8laToY-fDdURwaMDr4udUpNc3aBsVb3o,9951
-onesecondtrader-0.52.0.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
-onesecondtrader-0.52.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-onesecondtrader-0.52.0.dist-info/RECORD,,
+onesecondtrader/secmaster/schema_versions/secmaster_schema_v1.sql,sha256=E41rVhpYlXiC_GR4cw1bNQW_8Fdy8d-s1RJASIUCijM,12974
+onesecondtrader/secmaster/utils.py,sha256=d8PMSNLWVr10G0CSdL9vF-j_9jTfvOLxC-6k42x6LRU,19587
+onesecondtrader-0.54.0.dist-info/METADATA,sha256=iVpo6U1CIhN_NOYh1yzyqq25GVcadtP26VL93M6NehY,9951
+onesecondtrader-0.54.0.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+onesecondtrader-0.54.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+onesecondtrader-0.54.0.dist-info/RECORD,,

{onesecondtrader-0.52.0.dist-info → onesecondtrader-0.54.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.3.0
+Generator: poetry-core 2.3.1
 Root-Is-Purelib: true
 Tag: py3-none-any