onesecondtrader 0.51.0__tar.gz → 0.53.0__tar.gz

{onesecondtrader-0.51.0 → onesecondtrader-0.53.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: onesecondtrader
-Version: 0.51.0
+Version: 0.53.0
 Summary: The Trading Infrastructure Toolkit for Python. Research, simulate, and deploy algorithmic trading strategies — all in one place.
 License-File: LICENSE
 Author: Nils P. Kujath

{onesecondtrader-0.51.0 → onesecondtrader-0.53.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "onesecondtrader"
-version = "0.51.0"
+version = "0.53.0"
 description = "The Trading Infrastructure Toolkit for Python. Research, simulate, and deploy algorithmic trading strategies — all in one place."
 authors = [
     {name = "Nils P. Kujath",email = "63961429+NilsKujath@users.noreply.github.com"}
@@ -21,6 +21,7 @@ dependencies = [
 
 [tool.poetry]
 packages = [{include = "onesecondtrader", from = "src"}]
+include = ["src/onesecondtrader/**/*.sql"]
 
 
 [tool.poetry.group.dev.dependencies]

onesecondtrader-0.53.0/src/onesecondtrader/secmaster/__init__.py

@@ -0,0 +1,15 @@
+"""
+Provides a schema for creating and utilities to populate the security master database.
+"""
+
+from .utils import (
+    create_secmaster_db,
+    ingest_databento_zip,
+    ingest_databento_dbn,
+)
+
+__all__ = [
+    "create_secmaster_db",
+    "ingest_databento_zip",
+    "ingest_databento_dbn",
+]

onesecondtrader-0.53.0/src/onesecondtrader/secmaster/schema_versions/secmaster_schema_v1.sql

@@ -0,0 +1,197 @@
+-- Security master database schema.
+--
+-- The schema is designed for Databento-native ingestion via DBN files, while remaining compatible with other sources.
+-- Instrument identity is modeled per publisher namespace and supports either numeric upstream identifiers or symbols.
+-- Contract specifications and other static reference metadata are intentionally out of scope for this schema and should be stored separately if ingested.
+--
+-- The schema is explicitly ingestion-safe in the sense that:
+--
+-- 1) publishers are keyed by (vendor, dataset) rather than vendor alone, allowing multiple feeds per vendor;
+-- 2) symbology admits multiple mappings sharing the same start date by including the resolved instrument identifier
+--    in the primary key, preventing accidental overwrites during bulk ingestion.
+--
+-- | Table         | Description |
+-- |---------------|-------------|
+-- | `publishers`  | Registry of vendor+dataset namespaces used for market data and instrument ingestion. |
+-- | `instruments` | Registry of instruments observed from ingestion within a publisher namespace. |
+-- | `ohlcv`       | Aggregated OHLCV bar data keyed by instrument, bar duration (`rtype`), and event timestamp (`ts_event`). |
+-- | `symbology`   | Time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers. |
+
+
+
+-- Registry of all data sources used for market data and instrument ingestion.
+--
+-- Each row represents a distinct data product (feed) within a vendor namespace.
+-- A publisher record is uniquely identified by the pair (`name`, `dataset`), not by `name` alone.
+-- This allows a single vendor (e.g. Databento) to appear multiple times, once per concrete dataset/feed
+-- (e.g. `GLBX.MDP3`, `XNAS.ITCH`).
+--
+-- A publisher establishes the provenance of instrument definitions and price data and provides the context
+-- in which raw symbols and native instrument identifiers are interpreted.
+--
+-- | Field           | Type      | Constraints             | Description                                                                                                                                                                     |
+-- |-----------------|-----------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `publisher_id`  | `INTEGER` | `PRIMARY KEY`           | Internal surrogate key uniquely identifying a publisher record within the system.                                                                                               |
+-- | `name`          | `TEXT`    | `NOT NULL`              | Human-readable vendor identifier for the data source (e.g. `databento`, `yfinance`).                                                                                            |
+-- | `dataset`       | `TEXT`    | `NOT NULL`              | Identifier of the concrete data product or feed through which data is sourced; uses Databento dataset names (e.g. `GLBX.MDP3`) for Databento ingestion and internal identifiers for other sources (e.g. `YFINANCE`). |
+-- | `venue`         | `TEXT`    |                         | Optional ISO 10383 Market Identifier Code (MIC) describing the primary trading venue; may be NULL for aggregated or multi-venue sources.                                        |
+--
+-- **Table constraints**
+--
+-- * `UNIQUE(name, dataset)` ensures that each vendor+feed combination is represented at most once.
+--
+-- **Examples**
+--
+-- Databento CME Globex feed:
+--
+-- *  `name`    = `'databento'`
+-- *  `dataset` = `'GLBX.MDP3'`
+-- *  `venue`   = `XCME`
+--
+-- Databento NASDAQ TotalView feed:
+--
+-- *  `name`    = `'databento'`
+-- *  `dataset` = `'XNAS.ITCH'`
+-- *  `venue`   = `XNAS`
+--
+-- Yahoo Finance equity data:
+--
+-- *  `name`    = `'yfinance'`
+-- *  `dataset` = `'YFINANCE'`
+-- *  `venue`   = `NULL`
+--
+CREATE TABLE publishers (
+    publisher_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    dataset TEXT NOT NULL,
+    venue TEXT,
+    UNIQUE (name, dataset)
+);
+
+
+
+
+
+-- Registry of instruments observed through market data ingestion.
+--
+-- Each row represents an instrument identity within a publisher namespace.
+-- Instruments may be identified by a publisher-native numeric identifier, a symbol identifier, or both.
+-- Databento ingestion uses `source_instrument_id` as the primary identifier and may optionally store a symbol from symbology.
+-- Symbol-first sources such as yfinance use `symbol` as the primary identifier and typically leave `source_instrument_id` to be `NULL`.
+--
+-- The table does not store contract specifications or other reference metadata.
+-- Such metadata must be stored separately when available.
+--
+-- | Field                  | Type      | Constraints      | Description                                                                                                                                                  |
+-- |------------------------|-----------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id`        | `INTEGER` | `PRIMARY KEY`    | Internal surrogate key identifying an instrument record within the system.                                                                                   |
+-- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which this instrument identity is valid.                             |
+-- | `source_instrument_id` | `INTEGER` |                  | Publisher-native numeric instrument identifier as provided by the upstream data source (e.g. Databento instrument_id); may be `NULL` for symbol-only sources. |
+-- | `symbol`               | `TEXT`    |                  | Publisher-native symbol string identifying the instrument (e.g. raw symbol, ticker); may be NULL when numeric identifiers are used.                          |
+-- | `symbol_type`          | `TEXT`    |                  | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`, `ticker`).                                |
+--
+-- Each instrument must be identifiable by at least one of `source_instrument_id` or `symbol`.
+-- Uniqueness constraints ensure that instrument identities do not collide within a publisher namespace.
+-- The table intentionally excludes contract specifications and other reference metadata, which must be stored separately when available.
+--
+CREATE TABLE instruments (
+    instrument_id INTEGER PRIMARY KEY,
+
+    publisher_ref INTEGER NOT NULL,
+
+    source_instrument_id INTEGER,
+    symbol TEXT,
+	symbol_type TEXT,
+
+    FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
+
+    CHECK (
+        source_instrument_id IS NOT NULL
+        OR symbol IS NOT NULL
+    ),
+
+	CHECK (symbol IS NULL OR symbol_type IS NOT NULL),
+
+    UNIQUE (publisher_ref, source_instrument_id),
+    UNIQUE (publisher_ref, symbol, symbol_type)
+);
+
+
+
+
+
+
+-- Stores aggregated OHLCV bars for instruments at multiple time resolutions.
+--
+-- | Field           | Type      | Constraints                                 | Description                                                                                                              |
+-- |-----------------|-----------|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+-- | `instrument_id` | `INTEGER` | `NOT NULL`, `FK`                            | Foreign key reference to `instruments.instrument_id`, identifying the instrument to which this bar belongs.              |
+-- | `rtype`         | `INTEGER` | `NOT NULL`, `CHECK IN (32, 33, 34, 35, 36)` | Record type code encoding the bar duration using Databento OHLCV conventions (e.g. `32`=1s, `33`=1m, `34`=1h, `35`=1d). |
+-- | `ts_event`      | `INTEGER` | `NOT NULL`                                  | Event timestamp of the bar as provided by the upstream source, stored as nanoseconds since the UTC Unix epoch.           |
+-- | `open`          | `INTEGER` | `NOT NULL`                                  | Opening price of the bar interval, stored as a fixed-point integer using the upstream price scaling convention.          |
+-- | `high`          | `INTEGER` | `NOT NULL`                                  | Highest traded price during the bar interval, stored as a fixed-point integer.                                           |
+-- | `low`           | `INTEGER` | `NOT NULL`, `CHECK(low <= high)`            | Lowest traded price during the bar interval, stored as a fixed-point integer.                                            |
+-- | `close`         | `INTEGER` | `NOT NULL`                                  | Closing price of the bar interval, stored as a fixed-point integer.                                                      |
+-- | `volume`        | `INTEGER` | `NOT NULL`, `CHECK(volume >= 0)`            | Total traded volume during the bar interval.                                                                             |
+--
+-- The composite primary key enforces uniqueness per instrument, bar duration, and event timestamp.
+-- Integrity constraints ensure basic OHLC consistency and prevent invalid price relationships from being stored.
+-- The table uses `WITHOUT ROWID` to store rows directly in the primary key B-tree for reduced storage overhead and faster lookups.
+--
+CREATE TABLE ohlcv (
+    instrument_id INTEGER NOT NULL,
+    rtype INTEGER NOT NULL CHECK(rtype IN (32, 33, 34, 35, 36)),
+    ts_event INTEGER NOT NULL,
+    open INTEGER NOT NULL,
+    high INTEGER NOT NULL,
+    low INTEGER NOT NULL,
+    close INTEGER NOT NULL,
+    volume INTEGER NOT NULL CHECK(volume >= 0),
+    FOREIGN KEY (instrument_id) REFERENCES instruments(instrument_id),
+    PRIMARY KEY (instrument_id, rtype, ts_event),
+    CHECK(low <= high),
+    CHECK(open BETWEEN low AND high),
+    CHECK(close BETWEEN low AND high)
+) WITHOUT ROWID;
+
+
+
+
+
+-- Stores time-bounded mappings from publisher-native symbols to publisher-native instrument identifiers.
+--
+-- The table captures symbol resolution rules as provided by upstream data sources and must be interpreted within the
+-- namespace of a specific publisher.
+--
+-- The schema permits multiple mappings to share the same `start_date` for a given (`publisher_ref`, `symbol`, `symbol_type`)
+-- by including `source_instrument_id` in the primary key. This prevents accidental overwrite when upstream symbology exports
+-- contain same-day corrections, backfills, or parallel resolution segments.
+--
+-- | Field                  | Type      | Constraints      | Description                                                                                                                |
+-- |------------------------|-----------|------------------|----------------------------------------------------------------------------------------------------------------------------|
+-- | `publisher_ref`        | `INTEGER` | `NOT NULL`, `FK` | Foreign key reference to `publishers.publisher_id`, defining the publisher namespace in which the symbol mapping is valid. |
+-- | `symbol`               | `TEXT`    | `NOT NULL`       | Publisher-native symbol string as provided by the upstream source (e.g. raw symbol, continuous symbol).                    |
+-- | `symbol_type`          | `TEXT`    | `NOT NULL`       | Identifier describing the symbol scheme or resolution type used by the publisher (e.g. `raw_symbol`, `continuous`).        |
+-- | `source_instrument_id` | `INTEGER` | `NOT NULL`       | Publisher-native numeric instrument identifier corresponding to the resolved symbol.                                       |
+-- | `start_date`           | `TEXT`    | `NOT NULL`       | First calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.          |
+-- | `end_date`             | `TEXT`    | `NOT NULL`       | Last calendar date (inclusive) on which this symbol-to-instrument mapping is valid, stored in YYYY-MM-DD format.           |
+--
+-- The primary key enforces uniqueness of mappings at the granularity of a resolved instrument.
+-- Date bounds are interpreted as closed intervals.
+--
+CREATE TABLE symbology (
+    publisher_ref INTEGER NOT NULL,
+    symbol TEXT NOT NULL,
+    symbol_type TEXT NOT NULL,
+    source_instrument_id INTEGER NOT NULL,
+    start_date TEXT NOT NULL,
+    end_date TEXT NOT NULL,
+    FOREIGN KEY (publisher_ref) REFERENCES publishers(publisher_id),
+    FOREIGN KEY (publisher_ref, source_instrument_id)
+        REFERENCES instruments(publisher_ref, source_instrument_id),
+    PRIMARY KEY (publisher_ref, symbol, symbol_type, start_date, source_instrument_id),
+    CHECK (start_date <= end_date)
+);
+
+
+PRAGMA user_version = 1;

onesecondtrader-0.53.0/src/onesecondtrader/secmaster/utils.py

@@ -0,0 +1,578 @@
+from __future__ import annotations
+
+import json
+import logging
+import pathlib
+import shutil
+import sqlite3
+import tempfile
+import zipfile
+
+import databento
+
+
+BATCH_SIZE = 10000
+LOG_EVERY_OHLCV = 1_000_000
+LOG_EVERY_SYMBOLOGY = 50_000
+
+logger = logging.getLogger(__name__)
+
+
+def create_secmaster_db(db_path: pathlib.Path, schema_version: int = 1) -> pathlib.Path:
+    """
+    Create a new security master SQLite database using a selected schema version.
+
+    The database file is created at the given path and initialized by executing the SQL script
+    located in the `schema_versions` directory adjacent to this module.
+
+    The function expects the schema script to set `PRAGMA user_version` to the corresponding
+    schema version and verifies this after execution.
+
+    Parameters:
+        db_path:
+            Filesystem path at which the SQLite database file will be created.
+        schema_version:
+            Version number selecting the schema script to apply.
+
+    Returns:
+        The path to the created database file.
+
+    Raises:
+        FileExistsError:
+            If a file already exists at `db_path`.
+        FileNotFoundError:
+            If the schema script for `schema_version` does not exist.
+        sqlite3.DatabaseError:
+            If the applied schema does not set the expected `user_version` or if SQLite fails
+            while executing the schema.
+    """
+    if db_path.exists():
+        raise FileExistsError(f"Database already exists: {db_path}")
+
+    schema_path = (
+        pathlib.Path(__file__).resolve().parent
+        / "schema_versions"
+        / f"secmaster_schema_v{schema_version}.sql"
+    )
+
+    if not schema_path.is_file():
+        raise FileNotFoundError(
+            f"Schema version {schema_version} not found: {schema_path}"
+        )
+
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    schema_sql = schema_path.read_text(encoding="utf-8")
+
+    with sqlite3.connect(str(db_path)) as con:
+        con.execute("PRAGMA foreign_keys = ON;")
+        con.executescript(schema_sql)
+
+        row = con.execute("PRAGMA user_version;").fetchone()
+        actual_version = int(row[0]) if row else 0
+
+        if actual_version != schema_version:
+            raise sqlite3.DatabaseError(
+                f"Schema script set user_version={actual_version}, expected {schema_version}"
+            )
+
+    return db_path
+
+
+def ingest_databento_zip(
+    zip_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+    symbol_type: str = "raw_symbol",
+    dataset: str | None = None,
+) -> tuple[int, int]:
+    """
+    Ingest market data from a Databento zip archive into the security master database.
+
+    The archive may contain one or more DBN files and an optional `symbology.json`. The function
+    ingests OHLCV records from DBN files into `ohlcv` and ingests symbol-to-instrument mappings
+    into `symbology`.
+
+    The publisher namespace is created if absent. Publisher identity is determined by the pair
+    `(publisher_name, dataset)`, where `dataset` is extracted from `metadata.json` in the archive.
+
+    Ingestion is idempotent with respect to primary keys: existing `ohlcv` and `symbology` rows are
+    left unchanged.
+
+    Parameters:
+        zip_path:
+            Path to the Databento zip archive.
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from archive metadata.
+        symbol_type:
+            Symbol scheme stored in `symbology.symbol_type` for symbols found in `symbology.json`.
+        dataset:
+            Optional dataset override. If provided, it is used when `metadata.json` is missing or
+            does not specify a dataset.
+
+    Returns:
+        A tuple of (ohlcv_record_count_seen, symbology_record_count_seen).
+    """
+    ohlcv_count = 0
+    symbology_count = 0
+
+    logger.info("Opening Databento archive: %s", zip_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            with zipfile.ZipFile(zip_path, "r") as zf:
+                dataset, venue = _extract_dataset_info(zf, dataset_override=dataset)
+                logger.info(
+                    "Publisher resolved: name=%s dataset=%s venue=%s",
+                    publisher_name,
+                    dataset,
+                    venue,
+                )
+                publisher_id = _get_or_create_publisher(
+                    con, publisher_name, dataset, venue
+                )
+
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    dbn_files = [
+                        n
+                        for n in zf.namelist()
+                        if n.endswith(".dbn.zst") or n.endswith(".dbn")
+                    ]
+                    symbology_member = _zip_find_member(zf, "symbology.json")
+
+                    if not dbn_files and symbology_member is None:
+                        raise ValueError(
+                            "Archive contains no DBN files and no symbology.json"
+                        )
+
+                    logger.info("Found %d DBN file(s) in archive", len(dbn_files))
+
+                    for name in dbn_files:
+                        extracted_path = _zip_member_to_tempfile(zf, name, tmpdir)
+                        try:
+                            logger.info("Ingesting DBN file: %s", extracted_path.name)
+                            ohlcv_count += _ingest_dbn(
+                                extracted_path, con, publisher_id
+                            )
+                        finally:
+                            try:
+                                extracted_path.unlink()
+                            except FileNotFoundError:
+                                pass
+
+                    if symbology_member is not None:
+                        symbology_path = _zip_member_to_tempfile(
+                            zf, symbology_member, tmpdir
+                        )
+                        try:
+                            logger.info("Ingesting symbology.json")
+                            symbology_count += _ingest_symbology(
+                                symbology_path,
+                                con,
+                                publisher_id,
+                                symbol_type=symbol_type,
+                            )
+                        finally:
+                            try:
+                                symbology_path.unlink()
+                            except FileNotFoundError:
+                                pass
+                    else:
+                        logger.info("No symbology.json present in archive")
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info(
+        "Finished zip ingestion: %s (%d OHLCV records, %d symbology records)",
+        zip_path.name,
+        ohlcv_count,
+        symbology_count,
+    )
+
+    return ohlcv_count, symbology_count
+
+
+def ingest_databento_dbn(
+    dbn_path: pathlib.Path,
+    db_path: pathlib.Path,
+    publisher_name: str = "databento",
+) -> int:
+    """
+    Ingest market data from a Databento DBN file into the security master database.
+
+    Reads OHLCV records from the DBN file and inserts them into `ohlcv`. The publisher namespace
+    is created if absent. Publisher identity is determined by the pair `(publisher_name, dataset)`,
+    where `dataset` is read from DBN metadata.
+
+    Ingestion is idempotent with respect to primary keys: existing bars are left unchanged.
+
+    Parameters:
+        dbn_path:
+            Path to the DBN file (.dbn or .dbn.zst).
+        db_path:
+            Path to the security master SQLite database.
+        publisher_name:
+            Vendor name stored in `publishers.name`. The dataset is derived from DBN metadata.
+
+    Returns:
+        The number of OHLCV records seen in the DBN stream.
+    """
+    logger.info("Starting DBN ingestion: %s", dbn_path)
+
+    if not db_path.is_file():
+        raise FileNotFoundError(f"Security master DB not found: {db_path}")
+
+    con = sqlite3.connect(str(db_path))
+
+    try:
+        con.execute("PRAGMA foreign_keys = ON;")
+        _assert_secmaster_db(con)
+        _enable_bulk_loading(con)
+
+        with con:
+            store = databento.DBNStore.from_file(dbn_path)
+            dataset = store.metadata.dataset
+            if not dataset:
+                raise ValueError(f"DBN metadata missing dataset: {dbn_path}")
+            venue = dataset.split(".")[0] if "." in dataset else None
+
+            logger.info(
+                "Publisher resolved: name=%s dataset=%s venue=%s",
+                publisher_name,
+                dataset,
+                venue,
+            )
+
+            publisher_id = _get_or_create_publisher(con, publisher_name, dataset, venue)
+            count = _ingest_dbn(dbn_path, con, publisher_id)
+    finally:
+        try:
+            _disable_bulk_loading(con)
+        finally:
+            con.close()
+
+    logger.info("Finished DBN ingestion: %s (%d OHLCV records)", dbn_path.name, count)
+
+    return count
+
+
+def _extract_dataset_info(
+    zf: zipfile.ZipFile,
+    dataset_override: str | None = None,
+) -> tuple[str, str | None]:
+    metadata_member = _zip_find_member(zf, "metadata.json")
+    if metadata_member is None:
+        if dataset_override is None:
+            raise ValueError(
+                "Archive is missing metadata.json and no dataset override was provided"
+            )
+        dataset = dataset_override
+    else:
+        with zf.open(metadata_member) as f:
+            metadata = json.load(f)
+        dataset = metadata.get("query", {}).get("dataset")
+        if not dataset:
+            if dataset_override is None:
+                raise ValueError(
+                    f"metadata.json is missing query.dataset (member={metadata_member!r})"
+                )
+            dataset = dataset_override
+
+    venue = dataset.split(".")[0] if "." in dataset else None
+    return dataset, venue
+
+
+def _zip_find_member(
+    zf: zipfile.ZipFile,
+    basename: str,
+    allow_multiple: bool = False,
+) -> str | None:
+    candidates = [
+        name
+        for name in zf.namelist()
+        if name == basename or name.endswith("/" + basename)
+    ]
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        return candidates[0]
+
+    candidates = sorted(candidates)
+    if not allow_multiple:
+        raise ValueError(f"Multiple {basename} members found in archive: {candidates}")
+
+    selected = candidates[0]
+    logger.warning("Multiple %s found in archive; using %s", basename, selected)
+    return selected
+
+
+def _zip_member_to_tempfile(
+    zf: zipfile.ZipFile,
+    member_name: str,
+    tmpdir: str,
+) -> pathlib.Path:
+    suffix = "".join(pathlib.PurePosixPath(member_name).suffixes)
+    with tempfile.NamedTemporaryFile(
+        mode="wb",
+        suffix=suffix,
+        delete=False,
+        dir=tmpdir,
+    ) as tmp:
+        with zf.open(member_name) as src:
+            shutil.copyfileobj(src, tmp)
+        return pathlib.Path(tmp.name)
+
+
+def _get_or_create_publisher(
+    con: sqlite3.Connection,
+    name: str,
+    dataset: str,
+    venue: str | None,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT publisher_id FROM publishers WHERE name = ? AND dataset = ?",
+        (name, dataset),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO publishers (name, dataset, venue) VALUES (?, ?, ?)",
+        (name, dataset, venue),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _get_or_create_instrument(
+    con: sqlite3.Connection,
+    publisher_id: int,
+    source_instrument_id: int,
+) -> int:
+    cursor = con.cursor()
+    cursor.execute(
+        "SELECT instrument_id FROM instruments WHERE publisher_ref = ? AND source_instrument_id = ?",
+        (publisher_id, source_instrument_id),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    cursor.execute(
+        "INSERT INTO instruments (publisher_ref, source_instrument_id) VALUES (?, ?)",
+        (publisher_id, source_instrument_id),
+    )
+    return cursor.lastrowid  # type: ignore[return-value]
+
+
+def _assert_secmaster_db(
+    con: sqlite3.Connection, expected_user_version: int = 1
+) -> None:
+    row = con.execute("PRAGMA user_version;").fetchone()
+    user_version = int(row[0]) if row else 0
+    if user_version != expected_user_version:
+        raise sqlite3.DatabaseError(
+            "Security master schema user_version="
+            f"{user_version} does not match expected {expected_user_version}"
+        )
+
+    required = {"publishers", "instruments", "ohlcv", "symbology"}
+    present = {
+        r[0]
+        for r in con.execute(
+            "SELECT name FROM sqlite_master WHERE type = 'table'"
+        ).fetchall()
+    }
+    missing = sorted(required - present)
+    if missing:
+        raise sqlite3.DatabaseError(
+            f"Security master schema missing required tables: {', '.join(missing)}"
+        )
+
+
+def _ingest_dbn(
+    dbn_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+) -> int:
+    store = databento.DBNStore.from_file(dbn_path)
+    cursor = con.cursor()
+
+    instrument_cache: dict[int, int] = {}
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming OHLCV records from: %s", dbn_path.name)
+
+    for record in store:
+        if not isinstance(record, databento.OHLCVMsg):
+            continue
+
+        source_id = record.instrument_id
+        if source_id not in instrument_cache:
+            instrument_cache[source_id] = _get_or_create_instrument(
+                con, publisher_id, source_id
+            )
+        internal_id = instrument_cache[source_id]
+
+        rtype_val = (
+            record.rtype.value if hasattr(record.rtype, "value") else record.rtype
+        )
+
+        batch.append(
+            (
+                internal_id,
+                rtype_val,
+                record.ts_event,
+                record.open,
+                record.high,
+                record.low,
+                record.close,
+                record.volume,
+            )
+        )
+        count += 1
+
+        if count % LOG_EVERY_OHLCV == 0:
+            logger.info("Ingested %d OHLCV records from %s", count, dbn_path.name)
+
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                "INSERT OR IGNORE INTO ohlcv "
+                "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                batch,
+            )
+            batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO ohlcv "
+            "(instrument_id, rtype, ts_event, open, high, low, close, volume) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    logger.info("Completed OHLCV ingest from %s (%d records)", dbn_path.name, count)
+
+    return count
+
+
+def _ingest_symbology(
+    json_path: pathlib.Path,
+    con: sqlite3.Connection,
+    publisher_id: int,
+    symbol_type: str = "raw_symbol",
+) -> int:
+    if not isinstance(symbol_type, str) or not symbol_type:
+        raise ValueError("symbol_type must be a non-empty string")
+
+    with open(json_path, "r") as f:
+        data = json.load(f)
+
+    if not isinstance(data, dict):
+        raise ValueError("symbology.json root must be a JSON object")
+
+    result = data.get("result", {})
+    if not isinstance(result, dict):
+        raise ValueError("symbology.json['result'] must be an object")
+    cursor = con.cursor()
+
+    batch: list[tuple] = []
+    count = 0
+
+    logger.info("Streaming symbology mappings from: %s", json_path.name)
+
+    instrument_cache: set[int] = set()
+
+    for symbol, mappings in result.items():
+        if not isinstance(mappings, list):
+            raise ValueError(
+                f"symbology.json mappings must be a list for symbol={symbol!r}"
+            )
+
+        for i, mapping in enumerate(mappings):
+            if not isinstance(mapping, dict):
+                raise ValueError(
+                    f"symbology.json mapping must be an object at symbol={symbol!r} index={i}"
+                )
+
+            missing_keys = [k for k in ("s", "d0", "d1") if k not in mapping]
+            if missing_keys:
+                raise ValueError(
+                    "symbology.json mapping missing key(s) "
+                    f"{missing_keys} at symbol={symbol!r} index={i}"
+                )
+
+            source_id = int(mapping["s"])
+
+            if source_id not in instrument_cache:
+                _get_or_create_instrument(con, publisher_id, source_id)
+                instrument_cache.add(source_id)
+
+            batch.append(
+                (
+                    publisher_id,
+                    symbol,
+                    symbol_type,
+                    source_id,
+                    mapping["d0"],
+                    mapping["d1"],
+                )
+            )
+            count += 1
+
+            if count % LOG_EVERY_SYMBOLOGY == 0:
+                logger.info(
+                    "Ingested %d symbology mappings from %s", count, json_path.name
+                )
+
+            if len(batch) >= BATCH_SIZE:
+                cursor.executemany(
+                    "INSERT OR IGNORE INTO symbology "
+                    "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+                    "VALUES (?, ?, ?, ?, ?, ?)",
+                    batch,
+                )
+                batch.clear()
+
+    if batch:
+        cursor.executemany(
+            "INSERT OR IGNORE INTO symbology "
+            "(publisher_ref, symbol, symbol_type, source_instrument_id, start_date, end_date) "
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    logger.info(
+        "Completed symbology ingest from %s (%d mappings)", json_path.name, count
+    )
+
+    return count
+
+
+def _enable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA journal_mode = WAL")
+    con.execute("PRAGMA synchronous = NORMAL")
+    con.execute("PRAGMA cache_size = -64000")
+
+
+def _disable_bulk_loading(con: sqlite3.Connection) -> None:
+    con.execute("PRAGMA synchronous = FULL")
+    con.execute("PRAGMA journal_mode = DELETE")
+    con.execute("PRAGMA cache_size = -2000")