mxm-refdata 0.3.0__py3-none-any.whl

mxm/refdata/services/ref_data_service.py

@@ -0,0 +1,322 @@
+"""Service to create, update, and manage reference data in the database."""
+
+import logging
+from datetime import date
+
+from sqlalchemy.orm import DeclarativeBase
+
+from mxm.refdata.database.sql_session_manager import SQLSessionManager
+from mxm.refdata.mappings import (
+    futures_contract_to_orm,
+    futures_product_from_orm,
+    futures_product_to_orm,
+    period_from_orm,
+    period_to_orm,
+)
+from mxm.refdata.models import FuturesContract, Period
+from mxm.refdata.models.orm.futures_contracts import FuturesContractORM
+from mxm.refdata.models.orm.futures_products import FuturesProductORM
+from mxm.refdata.models.orm.period_cycles import (
+    PeriodCycleMembershipORM,
+    PeriodCycleORM,
+)
+from mxm.refdata.models.orm.periods import PeriodORM
+from mxm.refdata.models.period_cycles import CycleInstanceKind
+from mxm.refdata.models.periods import PeriodType
+from mxm.refdata.services.futures_contract_factory import FuturesContractFactory
+from mxm.refdata.services.futures_product_factory import FuturesProductFactory
+from mxm.refdata.services.period_factory import PeriodFactory
+from mxm.refdata.utils.config import load_config
+from mxm.refdata.utils.resources import futures_products_csv_path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+CYCLE_ID_CALENDAR_MONTHS = "CALENDAR_MONTHS"
+CYCLE_ID_CALENDAR_QUARTERS = "CALENDAR_QUARTERS"
+
+
+class RefDataService:
+    """
+    Manages reference data processes including initializing, updating, and resetting futures products.
+    """
+
+    def __init__(self, session_manager: SQLSessionManager):
+        """
+        Initialize the RefDataService with a session manager.
+
+        Args:
+            session_manager (SQLSessionManager): The session manager handling DB connections.
+        """
+        self.session_manager = session_manager
+        self.product_factory = FuturesProductFactory()
+        self.contract_factory = FuturesContractFactory()
+        self.period_factory = PeriodFactory()
+
+    def reset_database(self):
+        """
+        Reset the database by dropping and reinitializing all tables.
+        WARNING: This will delete all existing data.
+        """
+        logger.warning("Resetting the database. This will delete ALL existing data.")
+        self.session_manager.drop_db()
+        self.session_manager.init_db()
+        logger.info("Database successfully reset.")
+
+    def _is_database_empty(self) -> bool:
+        """
+        Check if the database contains any data in any relevant tables.
+
+        Returns:
+            bool: True if all relevant tables are empty, False otherwise.
+        """
+        with self.session_manager.db_session_scope() as session:
+            tables_to_check = [FuturesProductORM, FuturesContractORM, PeriodORM]
+
+            for table in tables_to_check:
+                if session.query(table).count() > 0:
+                    return False  # At least one table is not empty
+
+        return True  # All tables are empty
+
+    def is_table_empty(self, orm_model: type[DeclarativeBase]) -> bool:
+        """
+        Check if a given ORM table is empty.
+
+        Args:
+            orm_model (Base): The SQLAlchemy ORM model representing the table.
+
+        Returns:
+            bool: True if the table is empty, False otherwise.
+        """
+        with self.session_manager.db_session_scope() as session:
+            return session.query(orm_model).count() == 0
+
+    def setup_instruments(
+        self,
+        csv_file_path: str | None = None,
+        start_date: date | None = None,
+        end_date: date | None = None,
+    ) -> None:
+        """
+        Perform a full instrument setup: initialise periods, products, and contracts.
+
+        This method is a convenience function that ensures all required data is set up correctly.
+
+        NOTE: initialisation expects boundary-aligned horizons.
+        We filter to periods fully contained in [start_date, end_date] deliberately.
+        Do not call this with arbitrary dates; use a separate overlap query for ad hoc ranges.
+        Args:
+            csv_file_path (Optional[str]): Path to the CSV file for futures products.
+            start_date (Optional[date]): Start date for contract generation (default: today).
+            end_date (Optional[date]): End date for contract generation (default: +1 year).
+        """
+        logging.info("Starting full instrument setup...")
+
+        # Ensure we have a reasonable date range if not provided
+        start_date = start_date or date(2000, 1, 1)
+        end_date = end_date or date(2045, 12, 31)
+
+        # Step 1: Initialise periods (YEAR, QUARTER, MONTH)
+        self.initialise_periods(start_date=start_date, end_date=end_date)
+        self.initialise_period_cycles()
+        # Step 2: Initialise futures products from CSV
+        self.initialise_futures_products(csv_file_path=csv_file_path)
+
+        # Step 3: Generate and store contracts for each product
+        self.initialise_futures_contracts(start_date=start_date, end_date=end_date)
+
+        logging.info("Full instrument setup completed successfully.")
+
+    def initialise_futures_products(self, csv_file_path: str | None = None) -> None:
+        """
+        Initialize the database with futures products from a CSV file.
+
+        Args:
+            csv_file_path (Optional[str]): Path to the CSV file containing futures product definitions.
+                                        If None, uses the default path from config.
+
+        Raises:
+            ValueError: If the database is not empty.
+        """
+        if not self.is_table_empty(FuturesProductORM):
+            raise ValueError(
+                "Database already contains products. Run `reset_database()` first."
+            )
+
+        if csv_file_path is None:
+            cfg = load_config()
+            with futures_products_csv_path(cfg) as p:
+                products = FuturesProductFactory.initialise_from_csv(str(p))
+        else:
+            assert csv_file_path is not None
+            products = self.product_factory.initialise_from_csv(csv_file_path)
+
+        with self.session_manager.db_session_scope() as session:
+            for product in products:
+                product_orm = futures_product_to_orm(product)
+                session.add(product_orm)
+
+    def initialise_periods(
+        self,
+        start_date: date,
+        end_date: date,
+        period_types: list[PeriodType] | None = None,
+    ) -> None:
+        """
+        Initialise periods for a given date range and set of period types.
+
+        Ensures that all required periods exist **before** any contracts reference them.
+
+        Args:
+            start_date (date): The start date for period creation.
+            end_date (date): The end date for period creation.
+            period_types (Optional[list[PeriodType]]): List of period types to create.
+                Defaults to YEAR, QUARTER, and MONTH.
+        """
+        if not self.is_table_empty(PeriodORM):
+            raise ValueError(
+                "Database already contains periods. Run `reset_database()` first."
+            )
+
+        if period_types is None:
+            period_types = [PeriodType.YEAR, PeriodType.QUARTER, PeriodType.MONTH]
+
+        with self.session_manager.db_session_scope() as session:
+            existing_period_ids = {p.period_id for p in session.query(PeriodORM).all()}
+
+        new_periods: list[Period] = []
+        for period_type in period_types:
+            generated_periods = self.period_factory.get_periods_in_range(
+                start_date, end_date, period_type
+            )
+
+            for period in generated_periods:
+                if period.period_id not in existing_period_ids:
+                    new_periods.append(period)
+
+        if new_periods:
+            with self.session_manager.db_session_scope() as session:
+                for period in new_periods:
+                    session.add(
+                        period_to_orm(period)
+                    )  # Convert & store only new periods
+
+    def initialise_futures_contracts(self, start_date: date, end_date: date) -> None:
+        """
+        Initialize futures contracts using existing products and periods in the database.
+
+        Args:
+            start_date (date): The start date for contract generation.
+            end_date (date): The end date for contract generation.
+        """
+        with self.session_manager.db_session_scope() as session:
+            # Retrieve all existing products and convert them before session closes
+            products = [
+                futures_product_from_orm(productORM)
+                for productORM in session.query(FuturesProductORM).all()
+            ]
+            if not products:
+                raise ValueError(
+                    "No products found in the database. Run initialise_futures_products() first."
+                )
+
+            # Retrieve all periods within the date range
+            periods = {
+                p.period_id: period_from_orm(p)
+                for p in session.query(PeriodORM)
+                .filter(
+                    PeriodORM.first_date >= start_date, PeriodORM.last_date <= end_date
+                )
+                .all()
+            }
+            if not periods:
+                raise ValueError(
+                    "No periods found in the database. Run initialise_periods() first."
+                )
+
+        # Generate contracts outside of session
+        contracts: list[FuturesContract] = []
+        for product in products:
+            product_contracts = self.contract_factory.create_contracts_for_product(
+                product, periods
+            )
+            contracts.extend(product_contracts)
+
+        # Store contracts in the database
+        with self.session_manager.db_session_scope() as session:
+            for contract in contracts:
+                session.add(futures_contract_to_orm(contract=contract))
+
+    def initialise_period_cycles(self) -> None:
+        """
+        Initialise canonical PeriodCycles and PeriodCycleMemberships.
+
+        Requires PeriodORM to already exist.
+        """
+        if not self.is_table_empty(PeriodCycleORM) or not self.is_table_empty(
+            PeriodCycleMembershipORM
+        ):
+            raise ValueError(
+                "Database already contains period cycles. Run `reset_database()` first."
+            )
+
+        with self.session_manager.db_session_scope() as session:
+            # --- cycle definitions ---
+            session.add_all(
+                [
+                    PeriodCycleORM(
+                        cycle_id=CYCLE_ID_CALENDAR_MONTHS,
+                        name="Calendar Months",
+                        period_type=PeriodType.MONTH.name,
+                        instance_kind=CycleInstanceKind.YEAR.value,
+                        cycle_size=12,
+                    ),
+                    PeriodCycleORM(
+                        cycle_id=CYCLE_ID_CALENDAR_QUARTERS,
+                        name="Calendar Quarters",
+                        period_type=PeriodType.QUARTER.name,
+                        instance_kind=CycleInstanceKind.YEAR.value,
+                        cycle_size=4,
+                    ),
+                ]
+            )
+
+            # --- memberships derived from PeriodORM ---
+            periods = (
+                session.query(PeriodORM)
+                .filter(
+                    PeriodORM.period_type.in_(
+                        [PeriodType.MONTH.name, PeriodType.QUARTER.name]
+                    )
+                )
+                .all()
+            )
+
+            memberships: list[PeriodCycleMembershipORM] = []
+
+            for p in periods:
+                year = p.first_date.year
+                month = p.first_date.month
+
+                if p.period_type == PeriodType.MONTH:
+                    memberships.append(
+                        PeriodCycleMembershipORM(
+                            cycle_id=CYCLE_ID_CALENDAR_MONTHS,
+                            period_id=p.period_id,
+                            cycle_instance=year,
+                            cycle_element=month,  # 1..12
+                        )
+                    )
+                elif p.period_type == PeriodType.QUARTER:
+                    q = ((month - 1) // 3) + 1  # 1..4
+                    memberships.append(
+                        PeriodCycleMembershipORM(
+                            cycle_id=CYCLE_ID_CALENDAR_QUARTERS,
+                            period_id=p.period_id,
+                            cycle_instance=year,
+                            cycle_element=q,
+                        )
+                    )
+
+            session.add_all(memberships)

mxm/refdata/services/smokecheck.py

@@ -0,0 +1,326 @@
+"""Operational smoke checks for the materialised mxm-refdata database."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import Literal
+
+from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from mxm.refdata.database.sql_session_manager import SQLSessionManager
+from mxm.refdata.mappings import (
+    futures_contract_from_orm,
+    futures_product_from_orm,
+    period_from_orm,
+)
+from mxm.refdata.models.orm.futures_contracts import FuturesContractORM
+from mxm.refdata.models.orm.futures_products import FuturesProductORM
+from mxm.refdata.models.orm.period_cycles import (
+    PeriodCycleMembershipORM,
+    PeriodCycleORM,
+)
+from mxm.refdata.models.orm.periods import PeriodORM
+from mxm.refdata.models.periods import PeriodType
+from mxm.refdata.utils.period_types_codec import decode_period_types
+
+CYCLE_ID_CALENDAR_MONTHS = "CALENDAR_MONTHS"
+CYCLE_ID_CALENDAR_QUARTERS = "CALENDAR_QUARTERS"
+
+type SmokeCheckStatus = Literal["pass", "fail"]
+
+
+class SmokeCheckFailed(Exception):
+    """Raised when an individual smoke check fails."""
+
+
+@dataclass(frozen=True)
+class RefDataCounts:
+    """Row counts for the materialised refdata database."""
+
+    products: int
+    periods: int
+    contracts: int
+    cycles: int
+    memberships: int
+
+
+@dataclass(frozen=True)
+class SmokeCheckResult:
+    """Result of one smoke check."""
+
+    name: str
+    status: SmokeCheckStatus
+    message: str = ""
+
+
+@dataclass(frozen=True)
+class SmokeCheckReport:
+    """Aggregate smoke-check report."""
+
+    counts: RefDataCounts
+    results: list[SmokeCheckResult]
+
+    @property
+    def passed(self) -> bool:
+        """Return True if all smoke checks passed."""
+        return all(result.status == "pass" for result in self.results)
+
+
+type SmokeCheck = Callable[[Session], None]
+
+
+def _fail(message: str) -> None:
+    raise SmokeCheckFailed(message)
+
+
+def _assert(condition: bool, message: str) -> None:
+    if not condition:
+        _fail(message)
+
+
+def _pick_first[T](iterable: Iterable[T]) -> T | None:
+    for item in iterable:
+        return item
+    return None
+
+
+def count_refdata_rows(session: Session) -> RefDataCounts:
+    """Count key materialised refdata tables."""
+    return RefDataCounts(
+        products=session.query(FuturesProductORM).count(),
+        periods=session.query(PeriodORM).count(),
+        contracts=session.query(FuturesContractORM).count(),
+        cycles=session.query(PeriodCycleORM).count(),
+        memberships=session.query(PeriodCycleMembershipORM).count(),
+    )
+
+
+def smoke_check_non_empty_core_tables(session: Session) -> None:
+    """Verify that products, periods, and contracts are populated."""
+    counts = count_refdata_rows(session)
+
+    _assert(counts.products > 0, "No products inserted.")
+    _assert(counts.periods > 0, "No periods inserted.")
+    _assert(counts.contracts > 0, "No contracts inserted.")
+
+
+def smoke_check_roundtrip_period_types(session: Session) -> None:
+    """Verify period_types storage and ORM/domain round-trip semantics."""
+    product_orm = _pick_first(session.query(FuturesProductORM).limit(1).all())
+    if product_orm is None:
+        raise SmokeCheckFailed("No products found.")
+
+    decoded = decode_period_types(product_orm.period_types)
+    product = futures_product_from_orm(product_orm)
+
+    _assert(
+        product.period_types == decoded,
+        "Domain period_types does not match decoded ORM value.",
+    )
+
+
+def smoke_check_contract_date_types(session: Session) -> None:
+    """Verify contract date fields map correctly into the domain model."""
+    contract_orm = _pick_first(session.query(FuturesContractORM).limit(1).all())
+
+    if contract_orm is None:
+        raise SmokeCheckFailed("No contracts found.")
+
+    contract = futures_contract_from_orm(contract_orm)
+
+    _assert(
+        contract.first_day_of_interest == contract_orm.first_day_of_interest,
+        "Domain first_day_of_interest does not match ORM value.",
+    )
+    _assert(
+        contract.last_trading_day == contract_orm.last_trading_day,
+        "Domain last_trading_day does not match ORM value.",
+    )
+
+
+def smoke_check_contracts_periods_coherence(session: Session) -> None:
+    """Verify contracts reference periods and period-type filtering is feasible."""
+    product_id = _pick_first(
+        product_id
+        for (product_id,) in session.query(FuturesContractORM.product_id)
+        .distinct()
+        .limit(1)
+        .all()
+    )
+    _assert(product_id is not None, "No product_id found in contracts table.")
+
+    contracts_orm = (
+        session.query(FuturesContractORM)
+        .filter(FuturesContractORM.product_id == product_id)
+        .order_by(FuturesContractORM.contract_id.asc())
+        .all()
+    )
+    _assert(
+        len(contracts_orm) > 0,
+        f"No contracts found for product_id={product_id!r}.",
+    )
+
+    contracts = [futures_contract_from_orm(contract) for contract in contracts_orm]
+    periods = {p.period_id: period_from_orm(p) for p in session.query(PeriodORM).all()}
+
+    period_types = [
+        periods[contract.period_id].period_type
+        for contract in contracts
+        if contract.period_id in periods
+    ]
+
+    _assert(
+        len(period_types) > 0,
+        "Could not resolve period types for contracts via periods table.",
+    )
+
+    chosen = _pick_first(sorted(set(period_types), key=lambda item: item.value))
+    _assert(chosen is not None, "No period types found for product contracts.")
+
+    subset = [
+        contract
+        for contract in contracts
+        if periods[contract.period_id].period_type == chosen
+    ]
+    _assert(
+        0 < len(subset) <= len(contracts),
+        "Filtering by period_type should yield a non-empty subset.",
+    )
+
+
+def smoke_check_period_cycles_present(session: Session) -> None:
+    """Verify canonical calendar cycles exist and have memberships."""
+    cycle_ids = {
+        cycle_id for (cycle_id,) in session.query(PeriodCycleORM.cycle_id).all()
+    }
+
+    _assert(CYCLE_ID_CALENDAR_MONTHS in cycle_ids, "Missing cycle CALENDAR_MONTHS.")
+    _assert(CYCLE_ID_CALENDAR_QUARTERS in cycle_ids, "Missing cycle CALENDAR_QUARTERS.")
+
+    month_memberships = (
+        session.query(PeriodCycleMembershipORM)
+        .filter(PeriodCycleMembershipORM.cycle_id == CYCLE_ID_CALENDAR_MONTHS)
+        .count()
+    )
+    quarter_memberships = (
+        session.query(PeriodCycleMembershipORM)
+        .filter(PeriodCycleMembershipORM.cycle_id == CYCLE_ID_CALENDAR_QUARTERS)
+        .count()
+    )
+
+    _assert(month_memberships > 0, "No memberships for CALENDAR_MONTHS.")
+    _assert(quarter_memberships > 0, "No memberships for CALENDAR_QUARTERS.")
+
+
+def smoke_check_period_cycle_membership_uniqueness(session: Session) -> None:
+    """Verify canonical cycle membership keys are unique."""
+    duplicates = (
+        session.query(
+            PeriodCycleMembershipORM.cycle_id,
+            PeriodCycleMembershipORM.cycle_instance,
+            PeriodCycleMembershipORM.cycle_element,
+            func.count().label("n"),
+        )
+        .group_by(
+            PeriodCycleMembershipORM.cycle_id,
+            PeriodCycleMembershipORM.cycle_instance,
+            PeriodCycleMembershipORM.cycle_element,
+        )
+        .having(func.count() > 1)
+        .limit(1)
+        .all()
+    )
+
+    _assert(not duplicates, f"Duplicate cycle membership keys found: {duplicates!r}")
+
+
+def smoke_check_calendar_month_mapping(session: Session) -> None:
+    """Spot-check that December periods map to calendar month element 12."""
+    expected_month = 12
+    membership = (
+        session.query(PeriodCycleMembershipORM)
+        .filter(
+            PeriodCycleMembershipORM.cycle_id == CYCLE_ID_CALENDAR_MONTHS,
+            PeriodCycleMembershipORM.cycle_element == expected_month,
+        )
+        .limit(1)
+        .one_or_none()
+    )
+    if membership is None:
+        raise SmokeCheckFailed(
+            f"No month membership found for element={expected_month}."
+        )
+
+    period = (
+        session.query(PeriodORM)
+        .filter(PeriodORM.period_id == membership.period_id)
+        .one_or_none()
+    )
+    if period is None:
+        raise SmokeCheckFailed(
+            f"Membership references missing Period: {membership.period_id!r}"
+        )
+
+    _assert(
+        period.period_type == PeriodType.MONTH,
+        f"Expected mapped period_type MONTH, got {period.period_type!r}.",
+    )
+    _assert(
+        period.first_date.month == expected_month,
+        f"Expected Period.first_date.month={expected_month}, "
+        f"got {period.first_date.month}.",
+    )
+
+
+SMOKE_CHECKS: tuple[tuple[str, SmokeCheck], ...] = (
+    ("core tables populated", smoke_check_non_empty_core_tables),
+    ("period_types storage + round-trip", smoke_check_roundtrip_period_types),
+    ("contract date field types", smoke_check_contract_date_types),
+    (
+        "contracts/periods coherence + filterability",
+        smoke_check_contracts_periods_coherence,
+    ),
+    (
+        "period cycles present + non-empty memberships",
+        smoke_check_period_cycles_present,
+    ),
+    (
+        "period cycle membership uniqueness",
+        smoke_check_period_cycle_membership_uniqueness,
+    ),
+    ("calendar month mapping spot-check", smoke_check_calendar_month_mapping),
+)
+
+
+def run_smokechecks(
+    session_manager: SQLSessionManager | None = None,
+) -> SmokeCheckReport:
+    """Run operational smoke checks against the materialised refdata database."""
+    sm = session_manager or SQLSessionManager()
+
+    with sm.db_session_scope() as session:
+        counts = count_refdata_rows(session)
+        results: list[SmokeCheckResult] = []
+
+        for name, check in SMOKE_CHECKS:
+            try:
+                check(session)
+            except SmokeCheckFailed as err:
+                results.append(
+                    SmokeCheckResult(
+                        name=name,
+                        status="fail",
+                        message=str(err),
+                    )
+                )
+            else:
+                results.append(
+                    SmokeCheckResult(
+                        name=name,
+                        status="pass",
+                    )
+                )
+
+    return SmokeCheckReport(counts=counts, results=results)