onesecondtrader 0.52.0__tar.gz → 0.53.0__tar.gz

{onesecondtrader-0.52.0 → onesecondtrader-0.53.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: onesecondtrader
-Version: 0.52.0
+Version: 0.53.0
 Summary: The Trading Infrastructure Toolkit for Python. Research, simulate, and deploy algorithmic trading strategies — all in one place.
 License-File: LICENSE
 Author: Nils P. Kujath

{onesecondtrader-0.52.0 → onesecondtrader-0.53.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "onesecondtrader"
-version = "0.52.0"
+version = "0.53.0"
 description = "The Trading Infrastructure Toolkit for Python. Research, simulate, and deploy algorithmic trading strategies — all in one place."
 authors = [
     {name = "Nils P. Kujath",email = "63961429+NilsKujath@users.noreply.github.com"}

onesecondtrader-0.53.0/src/onesecondtrader/secmaster/__init__.py

@@ -0,0 +1,15 @@
+"""
+Provides a schema for creating and utilities to populate the security master database.
+"""
+
+from .utils import (
+    create_secmaster_db,
+    ingest_databento_zip,
+    ingest_databento_dbn,
+)
+
+__all__ = [
+    "create_secmaster_db",
+    "ingest_databento_zip",
+    "ingest_databento_dbn",
+]

{onesecondtrader-0.52.0 → onesecondtrader-0.53.0}/src/onesecondtrader/secmaster/schema_versions/secmaster_schema_v1.sql

@@ -4,10 +4,16 @@
 -- Instrument identity is modeled per publisher namespace and supports either numeric upstream identifiers or symbols.
 -- Contract specifications and other static reference metadata are intentionally out of scope for this schema and should be stored separately if ingested.
 --
+-- The schema is explicitly ingestion-safe in the sense that:
+--
+-- 1) publishers are keyed by (vendor, dataset) rather than vendor alone, allowing multiple feeds per vendor;
+-- 2) symbology admits multiple mappings sharing the same start date by including the resolved instrument identifier
+--    in the primary key, preventing accidental overwrites during bulk ingestion.
+--
 -- | Table         | Description |
 -- |---------------|-------------|
--- | `publishers`  | Registry of data sources and their identifier namespaces. |
--- | `instruments` | Registry of instruments observed from market data ingestion within a publisher namespace. |
+-- | `publishers`  | Registry of vendor+dataset namespaces used for market data and instrument ingestion. |
+-- | `instruments` | Registry of instruments observed from ingestion within a publisher namespace. |
 -- | `ohlcv`       | Aggregated OHLCV bar data keyed by instrument, bar duration (`rtype`), and event timestamp (`ts_event`). |
 -- | `symbology`   | Time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers. |
 
@@ -15,15 +21,24 @@
 
 -- Registry of all data sources used for market data and instrument ingestion.
 --
--- Each row represents a distinct data source.
--- A publisher establishes the provenance of instrument definitions and price data and provides the context in which raw symbols and native instrument identifiers are interpreted.
+-- Each row represents a distinct data product (feed) within a vendor namespace.
+-- A publisher record is uniquely identified by the pair (`name`, `dataset`), not by `name` alone.
+-- This allows a single vendor (e.g. Databento) to appear multiple times, once per concrete dataset/feed
+-- (e.g. `GLBX.MDP3`, `XNAS.ITCH`).
+--
+-- A publisher establishes the provenance of instrument definitions and price data and provides the context
+-- in which raw symbols and native instrument identifiers are interpreted.
 --
--- | Field           | Type      | Constraints          | Description                                                                                                                                                                     |
--- |-----------------|-----------|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
--- | `publisher_id`  | `INTEGER` | `PRIMARY KEY`        | Internal surrogate key uniquely identifying a data source within the system.                                                                                                    |
--- | `name`          | `TEXT`    | `NOT NULL`, `UNIQUE` | Human-readable identifier for the data source or vendor (e.g. `databento`, `yfinance`).                                                                                         |
--- | `dataset`       | `TEXT`    | `NOT NULL`           | Identifier of the concrete data product or feed through which data is sourced; uses Databento dataset names (e.g. `GLBX.MDP3`) for Databento ingestion and internal identifiers for other sources (e.g. `YFINANCE`). |
--- | `venue`         | `TEXT`    |                      | Optional ISO 10383 Market Identifier Code (MIC) describing the primary trading venue; may be NULL for aggregated or multi-venue sources.                                        |
+-- | Field           | Type      | Constraints             | Description                                                                                                                                                                     |
+-- |-----------------|-----------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `publisher_id`  | `INTEGER` | `PRIMARY KEY`           | Internal surrogate key uniquely identifying a publisher record within the system.                                                                                               |
+-- | `name`          | `TEXT`    | `NOT NULL`              | Human-readable vendor identifier for the data source (e.g. `databento`, `yfinance`).                                                                                            |
+-- | `dataset`       | `TEXT`    | `NOT NULL`              | Identifier of the concrete data product or feed through which data is sourced; uses Databento dataset names (e.g. `GLBX.MDP3`) for Databento ingestion and internal identifiers for other sources (e.g. `YFINANCE`). |
+-- | `venue`         | `TEXT`    |                         | Optional ISO 10383 Market Identifier Code (MIC) describing the primary trading venue; may be NULL for aggregated or multi-venue sources.                                        |
+--
+-- **Table constraints**
+--
+-- * `UNIQUE(name, dataset)` ensures that each vendor+feed combination is represented at most once.
 --
 -- **Examples**
 --
@@ -33,6 +48,12 @@
 -- *  `dataset` = `'GLBX.MDP3'`
 -- *  `venue`   = `XCME`
 --
+-- Databento NASDAQ TotalView feed:
+--
+-- *  `name`    = `'databento'`
+-- *  `dataset` = `'XNAS.ITCH'`
+-- *  `venue`   = `XNAS`
+--
 -- Yahoo Finance equity data:
 --
 -- *  `name`    = `'yfinance'`
@@ -41,14 +62,16 @@
 --
 CREATE TABLE publishers (
     publisher_id INTEGER PRIMARY KEY,
-    name TEXT NOT NULL UNIQUE,
+    name TEXT NOT NULL,
     dataset TEXT NOT NULL,
-    venue TEXT
+    venue TEXT,
+    UNIQUE (name, dataset)
 );
 
 
 
 
+
 -- Registry of instruments observed through market data ingestion.
 --
 -- Each row represents an instrument identity within a publisher namespace.
@@ -59,13 +82,13 @@ CREATE TABLE publishers (
 -- The table does not store contract specifications or other reference metadata.
 -- Such metadata must be stored separately when available.
 --
--- | Field                  | Type      | Constraints      | Description                                                                                                                                                 |
--- |------------------------|-----------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
--- | `instrument_id`        | `INTEGER` | `PRIMARY KEY`    | Internal surrogate key identifying an instrument record within the system.                                                                                  |
--- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which this instrument identity is valid.                            |
+-- | Field                  | Type      | Constraints      | Description                                                                                                                                                  |
+-- |------------------------|-----------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id`        | `INTEGER` | `PRIMARY KEY`    | Internal surrogate key identifying an instrument record within the system.                                                                                   |
+-- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which this instrument identity is valid.                             |
 -- | `source_instrument_id` | `INTEGER` |                  | Publisher-native numeric instrument identifier as provided by the upstream data source (e.g. Databento instrument_id); may be `NULL` for symbol-only sources. |
--- | `symbol`               | `TEXT`    |                  | Publisher-native symbol string identifying the instrument (e.g. raw symbol, ticker); may be NULL when numeric identifiers are used.                         |
--- | `symbol_type`          | `TEXT`    |                  | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`, `ticker`).                               |
+-- | `symbol`               | `TEXT`    |                  | Publisher-native symbol string identifying the instrument (e.g. raw symbol, ticker); may be NULL when numeric identifiers are used.                          |
+-- | `symbol_type`          | `TEXT`    |                  | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`, `ticker`).                                |
 --
 -- Each instrument must be identifiable by at least one of `source_instrument_id` or `symbol`.
 -- Uniqueness constraints ensure that instrument identities do not collide within a publisher namespace.
@@ -78,7 +101,7 @@ CREATE TABLE instruments (
 
     source_instrument_id INTEGER,
     symbol TEXT,
-    symbol_type TEXT,
+	symbol_type TEXT,
 
     FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
 
@@ -87,6 +110,8 @@ CREATE TABLE instruments (
         OR symbol IS NOT NULL
     ),
 
+	CHECK (symbol IS NULL OR symbol_type IS NOT NULL),
+
     UNIQUE (publisher_ref, source_instrument_id),
     UNIQUE (publisher_ref, symbol, symbol_type)
 );
@@ -98,16 +123,16 @@ CREATE TABLE instruments (
 
 -- Stores aggregated OHLCV bars for instruments at multiple time resolutions.
 --
--- | Field           | Type      | Constraints                                 | Description                                                                                                             |
--- |-----------------|-----------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
--- | `instrument_id` | `INTEGER` | `NOT NULL`, `FK`                            | Foreign key reference to `instruments.instrument_id`, identifying the instrument to which this bar belongs.             |
+-- | Field           | Type      | Constraints                                 | Description                                                                                                              |
+-- |-----------------|-----------|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id` | `INTEGER` | `NOT NULL`, `FK`                            | Foreign key reference to `instruments.instrument_id`, identifying the instrument to which this bar belongs.              |
 -- | `rtype`         | `INTEGER` | `NOT NULL`, `CHECK IN (32, 33, 34, 35, 36)` | Record type code encoding the bar duration using Databento OHLCV conventions (e.g. `32`=1s, `33`=1m, `34`=1h, `35`=1d). |
--- | `ts_event`      | `INTEGER` | `NOT NULL`                                  | Event timestamp of the bar as provided by the upstream source, stored as nanoseconds since the UTC Unix epoch.              |
--- | `open`          | `INTEGER` | `NOT NULL`                                  | Opening price of the bar interval, stored as a fixed-point integer using the upstream price scaling convention.         |
--- | `high`          | `INTEGER` | `NOT NULL`                                  | Highest traded price during the bar interval, stored as a fixed-point integer.                                          |
--- | `low`           | `INTEGER` | `NOT NULL`, `CHECK(low <= high)`            | Lowest traded price during the bar interval, stored as a fixed-point integer.                                           |
--- | `close`         | `INTEGER` | `NOT NULL`                                  | Closing price of the bar interval, stored as a fixed-point integer.                                                     |
--- | `volume`        | `INTEGER` | `NOT NULL`, `CHECK(volume >= 0)`            | Total traded volume during the bar interval.                                                                            |
+-- | `ts_event`      | `INTEGER` | `NOT NULL`                                  | Event timestamp of the bar as provided by the upstream source, stored as nanoseconds since the UTC Unix epoch.           |
+-- | `open`          | `INTEGER` | `NOT NULL`                                  | Opening price of the bar interval, stored as a fixed-point integer using the upstream price scaling convention.          |
+-- | `high`          | `INTEGER` | `NOT NULL`                                  | Highest traded price during the bar interval, stored as a fixed-point integer.                                           |
+-- | `low`           | `INTEGER` | `NOT NULL`, `CHECK(low <= high)`            | Lowest traded price during the bar interval, stored as a fixed-point integer.                                            |
+-- | `close`         | `INTEGER` | `NOT NULL`                                  | Closing price of the bar interval, stored as a fixed-point integer.                                                      |
+-- | `volume`        | `INTEGER` | `NOT NULL`, `CHECK(volume >= 0)`            | Total traded volume during the bar interval.                                                                             |
 --
 -- The composite primary key enforces uniqueness per instrument, bar duration, and event timestamp.
 -- Integrity constraints ensure basic OHLC consistency and prevent invalid price relationships from being stored.
@@ -135,7 +160,12 @@ CREATE TABLE ohlcv (
 
 -- Stores time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers.
 --
--- The table captures symbol resolution rules as provided by upstream data sources and must be interpreted within the namespace of a specific publisher.
+-- The table captures symbol resolution rules as provided by upstream data sources and must be interpreted within the
+-- namespace of a specific publisher.
+--
+-- The schema permits multiple mappings to share the same `start_date` for a given (`publisher_ref`, `symbol`, `symbol_type`)
+-- by including `source_instrument_id` in the primary key. This prevents accidental overwrite when upstream symbology exports
+-- contain same-day corrections, backfills, or parallel resolution segments.
 --
 -- | Field                  | Type      | Constraints      | Description                                                                                                                |
 -- |------------------------|-----------|------------------|----------------------------------------------------------------------------------------------------------------------------|
@@ -146,7 +176,7 @@ CREATE TABLE ohlcv (
 -- | `start_date`           | `TEXT`    | `NOT NULL`       | First calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.          |
 -- | `end_date`             | `TEXT`    | `NOT NULL`       | Last calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.           |
 --
--- The primary key enforces uniqueness of symbol mappings per publisher, symbol type, and start date.
+-- The primary key enforces uniqueness of mappings at the granularity of a resolved instrument.
 -- Date bounds are interpreted as closed intervals.
 --
 CREATE TABLE symbology (
@@ -157,8 +187,9 @@ CREATE TABLE symbology (
     start_date TEXT NOT NULL,
     end_date TEXT NOT NULL,
     FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
-    PRIMARY KEY (publisher_ref, symbol, symbol_type, start_date),
-    UNIQUE (publisher_ref, symbol_type, source_instrument_id, start_date),
+    FOREIGN KEY (publisher_ref, source_instrument_id)
+        REFERENCES instruments(publisher_ref, source_instrument_id),
+    PRIMARY KEY (publisher_ref, symbol, symbol_type, start_date, source_instrument_id),
     CHECK (start_date <= end_date)
 );
 

onesecondtrader-0.53.0/src/onesecondtrader/secmaster/utils.py

@@ -0,0 +1,578 @@
+from __future__ import annotations
+
+import json
+import logging
+import pathlib
+import shutil
+import sqlite3
+import tempfile
+import zipfile
+
+import databento
+
+
+BATCH_SIZE = 10000
+LOG_EVERY_OHLCV = 1_000_000
+LOG_EVERY_SYMBOLOGY = 50_000
+
+logger = logging.getLogger(__name__)
+
+
+def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathlib.Path:
+    """
+    Create a new security master SQLite database using a selected schema version.
+
+    The database file is created at the given path and initialized by executing the SQL script
+    located in the `schema_versions` directory adjacent to this module.
+
+    The function expects the schema script to set `PRAGMA user_version` to the corresponding
+    schema version and verifies this after execution.
+
+    Parameters:
+        db_path:
+            Filesystem path at which the SQLite database file will be created.
+        schema_version:
+            Version number selecting the schema script to apply.
+
+    Returns:
+        The path to the created database file.
+
+    Raises:
+        FileExistsError:
+            If a file already exists at `db_path`.
+        FileNotFoundError:
+            If the schema script for `schema_version` does not exist.
+        sqlite3.DatabaseError:
+            If the applied schema does not set the expected `user_version` or if SQLite fails
+            while executing the schema.
+    """
+    if db_path.exists():
+        raise FileExistsError(f"Database already exists: {db_path}")
+
+    schema_path = (
+        pathlib.Path(__file__).resolve().parent
+        / "schema_versions"
+        / f"secmaster_schema_v{schema_version}.sql"
+    )
+
+    if not schema_path.is_file():
+        raise FileNotFoundError(
+            f"Schema version {schema_version} not found: {schema_path}"
+        )
+
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    schema_sql = schema_path.read_text(encoding="utf-8")
+
+    with sqlite3.connect(str(db_path)) as con:
+        con.execute("PRAGMA foreign_keys = ON;")
+        con.executescript(schema_sql)
+
+        row = con.execute("PRAGMA user_version;").fetchone()
+        actual_version = int(row[0]) if row else 0
+
+        if actual_version != schema_version:
+            raise sqlite3.DatabaseError(
+                f"Schema script set user_version={actual_version}, expected {schema_version}"
+            )
+
+    return db_path
+
+
+def ingest_databento_zip(
+    zip_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+    symbol_type: str = "raw_symbol",
+    dataset: str | None = None,
+) -> tuple[int, int]:
+    """
+    Ingest market data from a Databento zip archive into the security master database.
+
+    The archive may contain one or more DBN files and an optional `symbology.json`. The function
+    ingests OHLCV records from DBN files into `ohlcv` and ingests symbol-to-instrument mappings
+    into `symbology`.
+
+    The publisher namespace is created if absent. Publisher identity is determined by the pair
+    `(publisher_name, dataset)`, where `dataset` is extracted from `metadata.json` in the archive.
+
+    Ingestion is idempotent with respect to primary keys: existing `ohlcv` and `symbology` rows are
+    left unchanged.
+
+    Parameters:
+        zip_path:
+            Path to the Databento zip archive.
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from archive metadata.
+        symbol_type:
+            Symbol scheme stored in `symbology.symbol_type` for symbols found in `symbology.json`.
+        dataset:
+            Optional dataset override. If provided, it is used when `metadata.json` is missing or
+            does not specify a dataset.
+
+    Returns:
+        A tuple of (ohlcv_record_count_seen, symbology_record_count_seen).
+    """
+    ohlcv_count = 0
+    symbology_count = 0
+
+    logger.info("Opening Databento archive: %s", zip_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            with zipfile.ZipFile(zip_path, "r") as zf:
+                dataset, venue = _extract_dataset_info(zf, dataset_override=dataset)
+                logger.info(
+                    "Publisher resolved: name=%s dataset=%s venue=%s",
+                    publisher_name,
+                    dataset,
+                    venue,
+                )
+                publisher_id = _get_or_create_publisher(
+                    con, publisher_name, dataset, venue
+                )
+
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    dbn_files = [
+                        n
+                        for n in zf.namelist()
+                        if n.endswith(".dbn.zst") or n.endswith(".dbn")
+                    ]
+                    symbology_member = _zip_find_member(zf, "symbology.json")
+
+                    if not dbn_files and symbology_member is None:
+                        raise ValueError(
+                            "Archive contains no DBN files and no symbology.json"
+                        )
+
+                    logger.info("Found %d DBN file(s) in archive", len(dbn_files))
+
+                    for name in dbn_files:
+                        extracted_path = _zip_member_to_tempfile(zf, name, tmpdir)
+                        try:
+                            logger.info("Ingesting DBN file: %s", extracted_path.name)
+                            ohlcv_count += _ingest_dbn(
+                                extracted_path, con, publisher_id
+                            )
+                        finally:
+                            try:
+                                extracted_path.unlink()
+                            except FileNotFoundError:
+                                pass
+
+                    if symbology_member is not None:
+                        symbology_path = _zip_member_to_tempfile(
+                            zf, symbology_member, tmpdir
+                        )
+                        try:
+                            logger.info("Ingesting symbology.json")
+                            symbology_count += _ingest_symbology(
+                                symbology_path,
+                                con,
+                                publisher_id,
+                                symbol_type=symbol_type,
+                            )
+                        finally:
+                            try:
+                                symbology_path.unlink()
+                            except FileNotFoundError:
+                                pass
+                    else:
+                        logger.info("No symbology.json present in archive")
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info(
+        "Finished zip ingestion: %s (%d OHLCV records, %d symbology records)",
+        zip_path.name,
+        ohlcv_count,
+        symbology_count,
+    )
+
+    return ohlcv_count, symbology_count
+
+
+def ingest_databento_dbn(
+    dbn_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+) -> int:
+    """
+    Ingest market data from a Databento DBN file into the security master database.
+
+    Reads OHLCV records from the DBN file and inserts them into `ohlcv`. The publisher namespace
+    is created if absent. Publisher identity is determined by the pair `(publisher_name, dataset)`,
+    where `dataset` is read from DBN metadata.
+
+    Ingestion is idempotent with respect to primary keys: existing bars are left unchanged.
+
+    Parameters:
+        dbn_path:
+            Path to the DBN file (.dbn or .dbn.zst).
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from DBN metadata.
+
+    Returns:
+        The number of OHLCV records seen in the DBN stream.
+    """
+    logger.info("Starting DBN ingestion: %s", dbn_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            store = databento.DBNStore.from_file(dbn_path)
+            dataset = store.metadata.dataset
+            if not dataset:
+                raise ValueError(f"DBN metadata missing dataset: {dbn_path}")
+            venue = dataset.split(".")[0] if "." in dataset else None
+
+            logger.info(
+                "Publisher resolved: name=%s dataset=%s venue=%s",
+                publisher_name,
+                dataset,
+                venue,
+            )
+
+            publisher_id = _get_or_create_publisher(con, publisher_name, dataset, venue)
+            count = _ingest_dbn(dbn_path, con, publisher_id)
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info("Finished DBN ingestion: %s (%d OHLCV records)", dbn_path.name, count)
+
+    return count
+
+
+def _extract_dataset_info(
+    zf: zipfile.ZipFile,
+    dataset_override: str | None = None,
+) -> tuple[str, str | None]:
+    metadata_member = _zip_find_member(zf, "metadata.json")
+    if metadata_member is None:
+        if dataset_override is None:
+            raise ValueError(
+                "Archive is missing metadata.json and no dataset override was provided"
+            )
+        dataset = dataset_override
+    else:
+        with zf.open(metadata_member) as f:
+            metadata = json.load(f)
+        dataset = metadata.get("query", {}).get("dataset")
+        if not dataset:
+            if dataset_override is None:
+                raise ValueError(
+                    f"metadata.json is missing query.dataset (member={metadata_member!r})"
+                )
+            dataset = dataset_override
+
+    venue = dataset.split(".")[0] if "." in dataset else None
+    return dataset, venue
+
+
+def _zip_find_member(
+    zf: zipfile.ZipFile,
+    basename: str,
+    allow_multiple: bool = False,
+) -> str | None:
+    candidates = [
+        name
+        for name in zf.namelist()
+        if name == basename or name.endswith("/" + basename)
+    ]
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        return candidates[0]
+
+    candidates = sorted(candidates)
+    if not allow_multiple:
+        raise ValueError(f"Multiple {basename} members found in archive: {candidates}")
+
+    selected = candidates[0]
+    logger.warning("Multiple %s found in archive; using %s", basename, selected)
+    return selected
+
+
+def _zip_member_to_tempfile(
+    zf: zipfile.ZipFile,
+    member_name: str,
+    tmpdir: str,
+) -> pathlib.Path:
+    suffix = "".join(pathlib.PurePosixPath(member_name).suffixes)
+    with tempfile.NamedTemporaryFile(
+        mode="wb",
+        suffix=suffix,
+        delete=False,
+        dir=tmpdir,
+    ) as tmp:
+        with zf.open(member_name) as src:
+            shutil.copyfileobj(src, tmp)
+        return pathlib.Path(tmp.name)
+
+
+def _get_or_create_publisher(
+    con: sqlite3.Connection,
+    name: str,
+    dataset: str,
+    venue: str | None,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT publisher_id FROM publishers WHERE name = ? AND dataset = ?",
+        (name, dataset),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO publishers (name, dataset, venue) VALUES (?, ?, ?)",
+        (name, dataset, venue),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _get_or_create_instrument(
+    con: sqlite3.Connection,
+    publisher_id: int,
+    source_instrument_id: int,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT instrument_id FROM instruments WHERE publisher_ref = ? AND source_instrument_id = ?",
+        (publisher_id, source_instrument_id),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO instruments (publisher_ref, source_instrument_id) VALUES (?, ?)",
+        (publisher_id, source_instrument_id),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _assert_secmaster_db(
+    con: sqlite3.Connection, expected_user_version: int = 1
+) -> None:
+    row = con.execute("PRAGMA user_version;").fetchone()
+    user_version = int(row[0]) if row else 0
+    if user_version != expected_user_version:
+        raise sqlite3.DatabaseError(
+            "Security master schema user_version="
+            f"{user_version} does not match expected {expected_user_version}"
+        )
+
+    required = {"publishers", "instruments", "ohlcv", "symbology"}
+    present = {
+        r[0]
+        for r in con.execute(
+            "SELECT name FROM sqlite_master WHERE type = 'table'"
+        ).fetchall()
+    }
+    missing = sorted(required - present)
+    if missing:
+        raise sqlite3.DatabaseError(
+            f"Security master schema missing required tables: {', '.join(missing)}"
+        )
+
+
+def _ingest_dbn(
+    dbn_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+) -> int:
+    store = databento.DBNStore.from_file(dbn_path)
+    cursor = con.cursor()
+
+    instrument_cache: dict[int, int] = {}
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming OHLCV records from: %s", dbn_path.name)
+
+    for record in store:
+        if not isinstance(record, databento.OHLCVMsg):
+            continue
+
+        source_id = record.instrument_id
+        if source_id not in instrument_cache:
+            instrument_cache[source_id] = _get_or_create_instrument(
+                con, publisher_id, source_id
+            )
+        internal_id = instrument_cache[source_id]
+
+        rtype_val = (
+            record.rtype.value if hasattr(record.rtype, "value") else record.rtype
+        )
+
+        batch.append(
+            (
+                internal_id,
+                rtype_val,
+                record.ts_event,
+                record.open,
+                record.high,
+                record.low,
+                record.close,
+                record.volume,
+            )
+        )
+        count += 1
+
+        if count % LOG_EVERY_OHLCV == 0:
+            logger.info("Ingested %d OHLCV records from %s", count, dbn_path.name)
+
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                "INSERT OR IGNORE INTO ohlcv "
+                "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                batch,
+            )
+            batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO ohlcv "
+            "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    logger.info("Completed OHLCV ingest from %s (%d records)", dbn_path.name, count)
+
+    return count
+
+
+def _ingest_symbology(
+    json_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+    symbol_type: str = "raw_symbol",
+) -> int:
+    if not isinstance(symbol_type, str) or not symbol_type:
+        raise ValueError("symbol_type must be a non-empty string")
+
+    with open(json_path, "r") as f:
+        data = json.load(f)
+
+    if not isinstance(data, dict):
+        raise ValueError("symbology.json root must be a JSON object")
+
+    result = data.get("result", {})
+    if not isinstance(result, dict):
+        raise ValueError("symbology.json['result'] must be an object")
+    cursor = con.cursor()
+
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming symbology mappings from: %s", json_path.name)
+
+    instrument_cache: set[int] = set()
+
+    for symbol, mappings in result.items():
+        if not isinstance(mappings, list):
+            raise ValueError(
+                f"symbology.json mappings must be a list for symbol={symbol!r}"
+            )
+
+        for i, mapping in enumerate(mappings):
+            if not isinstance(mapping, dict):
+                raise ValueError(
+                    f"symbology.json mapping must be an object at symbol={symbol!r} index={i}"
+                )
+
+            missing_keys = [k for k in ("s", "d0", "d1") if k not in mapping]
+            if missing_keys:
+                raise ValueError(
+                    "symbology.json mapping missing key(s) "
+                    f"{missing_keys} at symbol={symbol!r} index={i}"
+                )
+
+            source_id = int(mapping["s"])
+
+            if source_id not in instrument_cache:
+                _get_or_create_instrument(con, publisher_id, source_id)
+                instrument_cache.add(source_id)
+
+            batch.append(
+                (
+                    publisher_id,
+                    symbol,
+                    symbol_type,
+                    source_id,
+                    mapping["d0"],
+                    mapping["d1"],
+                )
+            )
+            count += 1
+
+            if count % LOG_EVERY_SYMBOLOGY == 0:
+                logger.info(
+                    "Ingested %d symbology mappings from %s", count, json_path.name
+                )
+
+            if len(batch) >= BATCH_SIZE:
+                cursor.executemany(
+                    "INSERT OR IGNORE INTO symbology "
+                    "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+                    "VALUES (?, ?, ?, ?, ?, ?)",
+                    batch,
+                )
+                batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO symbology "
+            "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    logger.info(
+        "Completed symbology ingest from %s (%d mappings)", json_path.name, count
+    )
+
+    return count
+
+
+def _enable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA journal_mode = WAL")
+    con.execute("PRAGMA synchronous = NORMAL")
+    con.execute("PRAGMA cache_size = -64000")
+
+
+def _disable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA synchronous = FULL")
+    con.execute("PRAGMA journal_mode = DELETE")
+    con.execute("PRAGMA cache_size = -2000")

onesecondtrader-0.52.0/src/onesecondtrader/secmaster/__init__.py

@@ -1,3 +0,0 @@
-"""
-Provides a schema for creating and utilities to populate the security master database.
-"""

onesecondtrader-0.52.0/src/onesecondtrader/secmaster/utils.py

@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-import pathlib
-import sqlite3
-
-
-def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathlib.Path:
-    """
-    Create a new security master SQLite database using a selected schema version.
-
-    The database file is created at the given path and initialized by executing the SQL script located in the `schema_versions` directory adjacent to this module.
-
-    The function expects the schema script to set `PRAGMA user_version` to the corresponding schema version and verifies this after execution.
-
-    Parameters:
-        db_path:
-            Filesystem path at which the SQLite database file will be created.
-        schema_version:
-            Version number selecting the schema script to apply.
-
-    Returns:
-        The path to the created database file.
-
-    Raises:
-        FileExistsError:
-            If a file already exists at `db_path`.
-        FileNotFoundError:
-            If the schema script for `schema_version` does not exist.
-        sqlite3.DatabaseError:
-            If the applied schema does not set the expected `user_version`
-            or if SQLite fails while executing the schema.
-    """
-    if db_path.exists():
-        raise FileExistsError(f"Database already exists: {db_path}")
-
-    schema_path = (
-        pathlib.Path(__file__).resolve().parent
-        / "schema_versions"
-        / f"secmaster_schema_v{schema_version}.sql"
-    )
-
-    if not schema_path.is_file():
-        raise FileNotFoundError(
-            f"Schema version {schema_version} not found: {schema_path}"
-        )
-
-    db_path.parent.mkdir(parents=True, exist_ok=True)
-
-    schema_sql = schema_path.read_text(encoding="utf-8")
-
-    with sqlite3.connect(str(db_path)) as con:
-        con.execute("PRAGMA foreign_keys = ON;")
-        con.executescript(schema_sql)
-
-        row = con.execute("PRAGMA user_version;").fetchone()
-        actual_version = int(row[0]) if row else 0
-
-        if actual_version != schema_version:
-            raise sqlite3.DatabaseError(
-                f"Schema script set user_version={actual_version}, "
-                f"expected {schema_version}"
-            )
-
-    return db_path