fixtureqa 0.1.0__py3-none-any.whl

fixture/core/fix_spec_parser.py

@@ -0,0 +1,172 @@
+"""
+Parse bundled FIX spec XMLs (FIX42.xml / FIX44.xml) into structured field and
+message definitions.  Results are cached in-process — parsed once on first use.
+"""
+
+from __future__ import annotations
+
+import os
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from typing import Optional
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FieldDef:
+    number: int
+    name: str
+    type: str                       # STRING, CHAR, INT, PRICE, etc.
+    values: dict[str, str]          # enum_value → description; empty if not enum
+
+
+@dataclass
+class FieldRef:
+    tag: int                        # 0 if unknown
+    name: str
+    type: str
+    required: bool
+    values: dict[str, str]
+
+
+@dataclass
+class MessageDef:
+    msg_type: str
+    name: str
+    category: str                   # 'admin' | 'app'
+    fields: list[FieldRef] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Internal cache
+# ---------------------------------------------------------------------------
+
+@dataclass
+class _Spec:
+    field_defs: dict[int, FieldDef]           # tag number → FieldDef
+    field_by_name: dict[str, FieldDef]        # name → FieldDef
+    messages: dict[str, MessageDef]           # msg_type → MessageDef
+
+
+_CACHE: dict[str, _Spec] = {}
+
+
+def _spec_path(begin_string: str) -> str:
+    here = os.path.dirname(os.path.abspath(__file__))
+    fname = "FIX42.xml" if "4.2" in begin_string else "FIX44.xml"
+    return os.path.abspath(os.path.join(here, "..", "fix_specs", fname))
+
+
+def _load_spec(begin_string: str) -> _Spec:
+    path = _spec_path(begin_string)
+    try:
+        tree = ET.parse(path)
+    except Exception as e:
+        logger.error("Failed to parse FIX spec %s: %s", path, e)
+        return _Spec(field_defs={}, field_by_name={}, messages={})
+
+    root = tree.getroot()
+
+    # ── 1. Parse <fields> section ──────────────────────────────────────────
+    field_defs: dict[int, FieldDef] = {}
+    field_by_name: dict[str, FieldDef] = {}
+
+    fields_el = root.find("fields")
+    if fields_el is not None:
+        for fel in fields_el.findall("field"):
+            try:
+                number = int(fel.get("number", "0"))
+            except ValueError:
+                continue
+            name = fel.get("name", "")
+            ftype = fel.get("type", "STRING")
+            values: dict[str, str] = {}
+            for vel in fel.findall("value"):
+                enum = vel.get("enum", "")
+                desc = vel.get("description", "")
+                if enum:
+                    values[enum] = desc
+            fd = FieldDef(number=number, name=name, type=ftype, values=values)
+            field_defs[number] = fd
+            field_by_name[name] = fd
+
+    # ── 2. Parse <messages> section ────────────────────────────────────────
+    messages: dict[str, MessageDef] = {}
+
+    messages_el = root.find("messages")
+    if messages_el is not None:
+        for mel in messages_el.findall("message"):
+            msg_type = mel.get("msgtype", "")
+            name = mel.get("name", "")
+            category = mel.get("msgcat", "app")
+            if not msg_type:
+                continue
+            refs: list[FieldRef] = _collect_field_refs(mel, field_by_name, required_override=None)
+            messages[msg_type] = MessageDef(
+                msg_type=msg_type,
+                name=name,
+                category=category,
+                fields=refs,
+            )
+
+    return _Spec(field_defs=field_defs, field_by_name=field_by_name, messages=messages)
+
+
+def _collect_field_refs(
+    el: ET.Element,
+    field_by_name: dict[str, FieldDef],
+    required_override: Optional[bool],
+) -> list[FieldRef]:
+    """Recursively collect field refs from <field> and <group> children."""
+    refs: list[FieldRef] = []
+    for child in el:
+        if child.tag == "field":
+            fname = child.get("name", "")
+            req = child.get("required", "N") == "Y"
+            if required_override is not None:
+                req = required_override
+            fd = field_by_name.get(fname)
+            refs.append(FieldRef(
+                tag=fd.number if fd else 0,
+                name=fname,
+                type=fd.type if fd else "STRING",
+                required=req,
+                values=fd.values if fd else {},
+            ))
+        elif child.tag == "group":
+            # Flatten group members; all members are optional (group itself may be required)
+            refs.extend(_collect_field_refs(child, field_by_name, required_override=False))
+    return refs
+
+
+def _get_spec(begin_string: str) -> _Spec:
+    if begin_string not in _CACHE:
+        _CACHE[begin_string] = _load_spec(begin_string)
+    return _CACHE[begin_string]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def get_messages(begin_string: str) -> list[MessageDef]:
+    """Return all message definitions for the given FIX version, sorted."""
+    spec = _get_spec(begin_string)
+    result = list(spec.messages.values())
+    result.sort(key=lambda m: (m.category != "app", m.name))
+    return result
+
+
+def get_message(begin_string: str, msg_type: str) -> MessageDef | None:
+    """Return the full message definition for one MsgType, or None."""
+    spec = _get_spec(begin_string)
+    return spec.messages.get(msg_type)
+
+
+def get_fields(begin_string: str) -> dict[int, FieldDef]:
+    """Return all field definitions keyed by tag number."""
+    spec = _get_spec(begin_string)
+    return spec.field_defs

fixture/core/fix_tags.py

@@ -0,0 +1,297 @@
+# FIX tag number -> field name (covers FIX 4.2 / 4.4 common fields)
+TAG_NAMES: dict[int, str] = {
+    1: "Account",
+    6: "AvgPx",
+    7: "BeginSeqNo",
+    8: "BeginString",
+    9: "BodyLength",
+    10: "CheckSum",
+    11: "ClOrdID",
+    12: "Commission",
+    13: "CommType",
+    14: "CumQty",
+    15: "Currency",
+    16: "EndSeqNo",
+    17: "ExecID",
+    18: "ExecInst",
+    19: "ExecRefID",
+    20: "ExecTransType",
+    21: "HandlInst",
+    22: "IDSource",
+    23: "IOIID",
+    24: "IOIOthSvc",
+    25: "IOIQltyInd",
+    26: "IOIRefID",
+    27: "IOIShares",
+    28: "IOITransType",
+    29: "LastCapacity",
+    30: "LastMkt",
+    31: "LastPx",
+    32: "LastShares",
+    33: "LinesOfText",
+    34: "MsgSeqNum",
+    35: "MsgType",
+    36: "NewSeqNo",
+    37: "OrderID",
+    38: "OrderQty",
+    39: "OrdStatus",
+    40: "OrdType",
+    41: "OrigClOrdID",
+    42: "OrigTime",
+    43: "PossDupFlag",
+    44: "Price",
+    45: "RefSeqNum",
+    48: "SecurityID",
+    49: "SenderCompID",
+    50: "SenderSubID",
+    52: "SendingTime",
+    54: "Side",
+    55: "Symbol",
+    56: "TargetCompID",
+    57: "TargetSubID",
+    58: "Text",
+    59: "TimeInForce",
+    60: "TransactTime",
+    61: "Urgency",
+    62: "ValidUntilTime",
+    63: "SettlmntTyp",
+    64: "FutSettDate",
+    65: "SymbolSfx",
+    66: "ListID",
+    67: "ListSeqNo",
+    68: "TotNoOrders",
+    69: "ListExecInst",
+    70: "AllocID",
+    73: "NoOrders",
+    74: "AvgPrxPrecision",
+    75: "TradeDate",
+    76: "ExecBroker",
+    77: "OpenClose",
+    78: "NoAllocs",
+    79: "AllocAccount",
+    80: "AllocShares",
+    81: "ProcessCode",
+    82: "NoRpts",
+    83: "RptSeq",
+    84: "CxlQty",
+    87: "AllocStatus",
+    88: "AllocRejCode",
+    89: "Signature",
+    90: "SecureDataLen",
+    91: "SecureData",
+    93: "SignatureLength",
+    94: "EmailType",
+    95: "RawDataLength",
+    96: "RawData",
+    97: "PossResend",
+    98: "EncryptMethod",
+    99: "StopPx",
+    100: "ExDestination",
+    102: "CxlRejReason",
+    103: "OrdRejReason",
+    104: "IOIQualifier",
+    107: "SecurityDesc",
+    108: "HeartBtInt",
+    109: "ClientID",
+    110: "MinQty",
+    111: "MaxFloor",
+    112: "TestReqID",
+    113: "ReportToExch",
+    114: "LocateReqd",
+    115: "OnBehalfOfCompID",
+    116: "OnBehalfOfSubID",
+    117: "QuoteID",
+    118: "NetMoney",
+    119: "SettlCurrAmt",
+    120: "SettlCurrency",
+    121: "ForexReq",
+    122: "OrigSendingTime",
+    123: "GapFillFlag",
+    124: "NoExecs",
+    126: "ExpireTime",
+    127: "DKReason",
+    128: "DeliverToCompID",
+    129: "DeliverToSubID",
+    130: "IOINaturalFlag",
+    131: "QuoteReqID",
+    132: "BidPx",
+    133: "OfferPx",
+    134: "BidSize",
+    135: "OfferSize",
+    136: "NoMiscFees",
+    137: "MiscFeeAmt",
+    138: "MiscFeeCurr",
+    139: "MiscFeeType",
+    140: "PrevClosePx",
+    141: "ResetSeqNumFlag",
+    142: "SenderLocationID",
+    143: "TargetLocationID",
+    144: "OnBehalfOfLocationID",
+    145: "DeliverToLocationID",
+    146: "NoRelatedSym",
+    147: "Subject",
+    148: "Headline",
+    149: "URLLink",
+    150: "ExecType",
+    151: "LeavesQty",
+    152: "CashOrderQty",
+    153: "AllocAvgPx",
+    154: "AllocNetMoney",
+    155: "SettlCurrFxRate",
+    156: "SettlCurrFxRateCalc",
+    157: "NumDaysInterest",
+    158: "AccruedInterestRate",
+    159: "AccruedInterestAmt",
+    160: "SettlInstMode",
+    161: "AllocText",
+    162: "SettlInstID",
+    163: "SettlInstTransType",
+    164: "EmailThreadID",
+    165: "SettlInstSource",
+    166: "SettlLocation",
+    167: "SecurityType",
+    168: "EffectiveTime",
+    169: "StandInstDbType",
+    170: "StandInstDbName",
+    171: "StandInstDbID",
+    172: "SettlDeliveryType",
+    173: "SettlDepositoryCode",
+    174: "SettlBrkrCode",
+    175: "SettlInstCode",
+    176: "SecuritySettlAgentName",
+    177: "SecuritySettlAgentCode",
+    178: "SecuritySettlAgentAcctNum",
+    179: "SecuritySettlAgentAcctName",
+    180: "SecuritySettlAgentContactName",
+    181: "SecuritySettlAgentContactPhone",
+    182: "CashSettlAgentName",
+    183: "CashSettlAgentCode",
+    184: "CashSettlAgentAcctNum",
+    185: "CashSettlAgentAcctName",
+    186: "CashSettlAgentContactName",
+    187: "CashSettlAgentContactPhone",
+    188: "BidSpotRate",
+    189: "BidForwardPoints",
+    190: "OfferSpotRate",
+    191: "OfferForwardPoints",
+    192: "OrderQty2",
+    193: "FutSettDate2",
+    194: "LastSpotRate",
+    195: "LastForwardPoints",
+    196: "AllocLinkID",
+    197: "AllocLinkType",
+    198: "SecondaryOrderID",
+    199: "NoIOIQualifiers",
+    200: "MaturityMonthYear",
+    201: "PutOrCall",
+    202: "StrikePrice",
+    203: "CoveredOrUncovered",
+    204: "CustomerOrFirm",
+    205: "MaturityDay",
+    206: "OptAttribute",
+    207: "SecurityExchange",
+    208: "NotifyBrokerOfCredit",
+    209: "AllocHandlInst",
+    210: "MaxShow",
+    211: "PegDifference",
+    212: "XmlDataLen",
+    213: "XmlData",
+    214: "SettlInstRefID",
+    215: "NoRoutingIDs",
+    216: "RoutingType",
+    217: "RoutingID",
+    # FIX 4.4 additions
+    369: "LastMsgSeqNumProcessed",
+    371: "RefTagID",
+    372: "RefMsgType",
+    373: "SessionRejectReason",
+    374: "BidRequestTransType",
+    375: "ContraBroker",
+    376: "ComplianceID",
+    377: "SolicitedFlag",
+    378: "ExecRestatementReason",
+    379: "BusinessRejectRefID",
+    380: "BusinessRejectReason",
+    381: "GrossTradeAmt",
+    382: "NoContraBrokers",
+    383: "MaxMessageSize",
+    384: "NoMsgTypes",
+    385: "MsgDirection",
+    386: "NoTradingSessions",
+    387: "TotalVolumeTraded",
+    388: "DiscretionInst",
+    389: "DiscretionOffset",
+    390: "BidID",
+    391: "ClientBidID",
+    392: "ListName",
+    393: "TotalNumSecurities",
+    394: "BidType",
+    395: "NumTickets",
+    396: "SideValue1",
+    397: "SideValue2",
+    398: "NoBidDescriptors",
+    399: "BidDescriptorType",
+    400: "BidDescriptor",
+    553: "Username",
+    554: "Password",
+}
+
+# MsgType (tag 35) value -> readable name
+MSG_TYPE_NAMES: dict[str, str] = {
+    "0": "Heartbeat",
+    "1": "TestRequest",
+    "2": "ResendRequest",
+    "3": "Reject",
+    "4": "SequenceReset",
+    "5": "Logout",
+    "6": "IndicationOfInterest",
+    "7": "Advertisement",
+    "8": "ExecutionReport",
+    "9": "OrderCancelReject",
+    "A": "Logon",
+    "B": "News",
+    "C": "Email",
+    "D": "NewOrderSingle",
+    "E": "NewOrderList",
+    "F": "OrderCancelRequest",
+    "G": "OrderCancelReplaceRequest",
+    "H": "OrderStatusRequest",
+    "J": "Allocation",
+    "K": "ListCancelRequest",
+    "L": "ListExecute",
+    "M": "ListStatusRequest",
+    "N": "ListStatus",
+    "P": "AllocationAck",
+    "Q": "DontKnowTrade",
+    "R": "QuoteRequest",
+    "S": "Quote",
+    "T": "SettlementInstructions",
+    "V": "MarketDataRequest",
+    "W": "MarketDataSnapshotFullRefresh",
+    "X": "MarketDataIncrementalRefresh",
+    "Y": "MarketDataRequestReject",
+    "Z": "QuoteCancel",
+    "a": "QuoteStatusRequest",
+    "b": "MassQuoteAcknowledgement",
+    "c": "SecurityDefinitionRequest",
+    "d": "SecurityDefinition",
+    "e": "SecurityStatusRequest",
+    "f": "SecurityStatus",
+    "g": "TradingSessionStatusRequest",
+    "h": "TradingSessionStatus",
+    "i": "MassQuote",
+    "j": "BusinessMessageReject",
+    "k": "BidRequest",
+    "l": "BidResponse",
+    "m": "ListStrikePrice",
+}
+
+
+def tag_name(tag: int) -> str:
+    """Return the field name for a tag number, or 'Tag<N>' if unknown."""
+    return TAG_NAMES.get(tag, f"Tag{tag}")
+
+
+def msg_type_name(msg_type: str) -> str:
+    """Return the human-readable name for a MsgType value."""
+    return MSG_TYPE_NAMES.get(msg_type, f"Unknown({msg_type})")

fixture/core/housekeeping.py

@@ -0,0 +1,107 @@
+"""
+Housekeeping service: age-based cleanup of SQLite messages and QuickFIX log files.
+
+One instance is created at app startup and shared. It starts a daemon background
+thread that runs cleanup once on startup (after a grace period) then every 24 h.
+It can also be triggered synchronously via run_now() from an API endpoint.
+"""
+
+import logging
+import os
+import sqlite3
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .user_store import UserStore
+
+logger = logging.getLogger(__name__)
+
+
+class HousekeepingService:
+    """
+    Cleans up old messages from SQLite and old QuickFIX .log files.
+
+    Settings are read from UserStore on each run so that changes made in the
+    admin UI take effect on the next scheduled run without a restart.
+    """
+
+    def __init__(self, msg_db_path: str, log_dir: str, user_store: "UserStore"):
+        self._msg_db_path = msg_db_path
+        self._log_dir = log_dir
+        self._user_store = user_store
+
+    def start(self) -> None:
+        """Start the background daemon thread."""
+        t = threading.Thread(target=self._loop, daemon=True, name="housekeeping")
+        t.start()
+
+    def run_now(self) -> dict:
+        """
+        Run cleanup synchronously. Returns {"msgs_deleted": N, "logs_deleted": N}.
+        Safe to call from any thread (used by the API endpoint).
+        """
+        settings = self._load_settings()
+        return {
+            "msgs_deleted": self._purge_messages(settings["msg_retention_days"]),
+            "logs_deleted": self._purge_logs(settings["log_retention_days"]),
+        }
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _load_settings(self) -> dict:
+        us = self._user_store
+        return {
+            "enabled": us.get_setting("housekeeping_enabled", "1") == "1",
+            "msg_retention_days": int(us.get_setting("msg_retention_days", "90")),
+            "log_retention_days": int(us.get_setting("log_retention_days", "30")),
+        }
+
+    def _loop(self) -> None:
+        time.sleep(30)  # startup grace period — let sessions initialise first
+        while True:
+            try:
+                settings = self._load_settings()
+                if settings["enabled"]:
+                    result = self.run_now()
+                    if result["msgs_deleted"] or result["logs_deleted"]:
+                        logger.info(
+                            "deleted %d messages, %d log files",
+                            result["msgs_deleted"], result["logs_deleted"],
+                        )
+            except Exception as exc:
+                logger.error("housekeeping error: %s", exc)
+            time.sleep(86400)  # 24 h
+
+    def _purge_messages(self, retention_days: int) -> int:
+        if retention_days == 0:
+            return 0
+        if not os.path.isfile(self._msg_db_path):
+            return 0
+        cutoff = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).isoformat()
+        with sqlite3.connect(self._msg_db_path, timeout=30) as conn:
+            cur = conn.execute("DELETE FROM messages WHERE ts < ?", (cutoff,))
+        return cur.rowcount
+
+    def _purge_logs(self, retention_days: int) -> int:
+        if retention_days == 0:
+            return 0
+        if not os.path.isdir(self._log_dir):
+            return 0
+        cutoff = time.time() - retention_days * 86400
+        count = 0
+        for root, _, files in os.walk(self._log_dir):
+            for fname in files:
+                if fname.endswith(".log"):
+                    path = os.path.join(root, fname)
+                    try:
+                        if os.path.getmtime(path) < cutoff:
+                            os.remove(path)
+                            count += 1
+                    except OSError:
+                        pass
+        return count

fixture/core/message_log.py

@@ -0,0 +1,115 @@
+"""
+Per-session message log with structured entries and filtering.
+"""
+
+import threading
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Optional
+
+from .fix_parser import parse_raw, get_msg_type, get_seq_num
+from .fix_tags import msg_type_name
+
+DEFAULT_MAX_ENTRIES = 10_000
+
+
+@dataclass
+class LogEntry:
+    direction: str          # "IN" or "OUT"
+    admin: bool             # True = session-level (Logon, Heartbeat, etc.)
+    raw: str                # raw FIX string (SOH-delimited)
+    timestamp: datetime     # UTC capture time
+    seq_num: int            # MsgSeqNum (tag 34); 0 if absent
+    msg_type: str           # raw MsgType (tag 35), e.g. "D", "8", "A"
+    msg_type_name: str      # human-readable, e.g. "NewOrderSingle"
+    fields: dict = field(default_factory=dict)  # {tag_int: value_str}
+    id: Optional[int] = None  # SQLite rowid; None for in-memory entries
+
+    def pretty(self, use_names: bool = True) -> str:
+        """Multi-line human-readable display."""
+        from .fix_parser import format_fields
+        lines = [
+            f"[{self.msg_type_name}]  {self.direction}  "
+            f"SeqNum={self.seq_num}  "
+            f"{self.timestamp.strftime('%H:%M:%S.%f')[:-3]}"
+        ]
+        for label, value in format_fields(self.fields, use_names=use_names):
+            lines.append(f"  {label} = {value}")
+        return "\n".join(lines)
+
+
+class MessageLog:
+    """
+    Thread-safe message log for one FIX session.
+
+    Messages arrive from the QuickFIX I/O thread; reads happen from the
+    UI or CLI thread — all access is protected by a lock.
+
+    When max_entries is reached, oldest entries are dropped (ring-buffer
+    behaviour) to prevent unbounded memory growth.
+    """
+
+    def __init__(self, max_entries: int = DEFAULT_MAX_ENTRIES):
+        self._entries: list[LogEntry] = []
+        self._max = max_entries
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Write
+    # ------------------------------------------------------------------
+
+    def add(self, direction: str, admin: bool, raw: str) -> LogEntry:
+        """Parse and store a message. Returns the created LogEntry."""
+        fields = parse_raw(raw)
+        mt = get_msg_type(fields)
+        entry = LogEntry(
+            direction=direction,
+            admin=admin,
+            raw=raw,
+            timestamp=datetime.now(tz=timezone.utc),
+            seq_num=get_seq_num(fields),
+            msg_type=mt,
+            msg_type_name=msg_type_name(mt) if mt else "Unknown",
+            fields=fields,
+        )
+        with self._lock:
+            if len(self._entries) >= self._max:
+                self._entries = self._entries[self._max // 10:]  # drop oldest 10%
+            self._entries.append(entry)
+        return entry
+
+    def clear(self) -> None:
+        with self._lock:
+            self._entries.clear()
+
+    # ------------------------------------------------------------------
+    # Read
+    # ------------------------------------------------------------------
+
+    def get_entries(
+        self,
+        *,
+        direction: Optional[str] = None,        # "IN" or "OUT"
+        admin: Optional[bool] = None,            # True/False to filter; None = both
+        msg_type: Optional[str] = None,          # raw MsgType value, e.g. "D"
+        limit: Optional[int] = None,             # return last N entries
+        before_id: Optional[int] = None,         # ignored for in-memory log (no DB rowids)
+    ) -> list[LogEntry]:
+        """Return a snapshot (copy) of entries matching the given filters."""
+        with self._lock:
+            entries = list(self._entries)
+
+        if direction is not None:
+            entries = [e for e in entries if e.direction == direction]
+        if admin is not None:
+            entries = [e for e in entries if e.admin == admin]
+        if msg_type is not None:
+            entries = [e for e in entries if e.msg_type == msg_type]
+        if limit is not None:
+            entries = entries[-limit:]
+
+        return entries
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._entries)