fixtureqa 0.1.0__py3-none-any.whl

fixture/core/message_store.py

@@ -0,0 +1,246 @@
+"""
+SQLite-backed message persistence.
+
+MessageWriter  — one shared instance per app; owns the background writer thread.
+MessageStore   — per-session interface; same API as MessageLog (drop-in replacement).
+
+The QuickFIX I/O thread calls MessageStore.add() which parses the message
+synchronously and enqueues the DB write. The background thread batches
+inserts so the I/O thread is never blocked by SQLite.
+"""
+
+import json
+import queue
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Optional
+
+from .fix_parser import parse_raw, get_msg_type, get_seq_num
+from .fix_tags import msg_type_name
+from .message_log import LogEntry
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS messages (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id    TEXT    NOT NULL,
+    ts            TEXT    NOT NULL,
+    direction     TEXT    NOT NULL,
+    admin         INTEGER NOT NULL,
+    seq_num       INTEGER NOT NULL,
+    msg_type      TEXT    NOT NULL,
+    msg_type_name TEXT    NOT NULL,
+    raw           TEXT    NOT NULL,
+    fields_json   TEXT    NOT NULL
+)
+"""
+_CREATE_INDEX = """
+CREATE INDEX IF NOT EXISTS idx_msg_session ON messages(session_id, id DESC)
+"""
+_INSERT = """
+INSERT INTO messages
+    (session_id, ts, direction, admin, seq_num, msg_type, msg_type_name, raw, fields_json)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+"""
+
+
+def _entry_to_row(session_id: str, entry: LogEntry) -> tuple:
+    return (
+        session_id,
+        entry.timestamp.isoformat(),
+        entry.direction,
+        1 if entry.admin else 0,
+        entry.seq_num,
+        entry.msg_type,
+        entry.msg_type_name,
+        entry.raw,
+        json.dumps(entry.fields),
+    )
+
+
+def _row_to_entry(row) -> LogEntry:
+    row_id, _, ts, direction, admin, seq_num, mt, mt_name, raw, fields_json = row
+    return LogEntry(
+        direction=direction,
+        admin=bool(admin),
+        raw=raw,
+        timestamp=datetime.fromisoformat(ts),
+        seq_num=seq_num,
+        msg_type=mt,
+        msg_type_name=mt_name,
+        fields=json.loads(fields_json),
+        id=row_id,
+    )
+
+
+class MessageWriter:
+    """
+    Shared background SQLite writer for all sessions.
+
+    One instance per application. All MessageStore instances for the same
+    app share this writer. The background thread batches inserts to keep
+    write throughput high without blocking the QuickFIX I/O thread.
+    """
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+        self._queue: queue.Queue = queue.Queue()
+        self._init_db()
+        self._thread = threading.Thread(
+            target=self._run, name="msg-writer", daemon=True
+        )
+        self._thread.start()
+
+    def _init_db(self) -> None:
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA synchronous=NORMAL")
+            conn.execute(_CREATE_TABLE)
+            conn.execute(_CREATE_INDEX)
+
+    def enqueue(self, session_id: str, entry: LogEntry) -> None:
+        """Called from any thread. Never blocks."""
+        self._queue.put((session_id, entry))
+
+    def close(self) -> None:
+        """Flush remaining writes and stop the background thread."""
+        self._queue.put(None)  # sentinel
+        self._thread.join(timeout=10.0)
+
+    def _run(self) -> None:
+        conn = sqlite3.connect(self._db_path)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA synchronous=NORMAL")
+        stop = False
+        try:
+            while not stop:
+                batch = []
+                # Block up to 100 ms waiting for the first item
+                try:
+                    item = self._queue.get(timeout=0.1)
+                    if item is None:
+                        stop = True
+                    else:
+                        batch.append(item)
+                        # Drain additional items without blocking
+                        while len(batch) < 200:
+                            try:
+                                item = self._queue.get_nowait()
+                                if item is None:
+                                    stop = True
+                                    break
+                                batch.append(item)
+                            except queue.Empty:
+                                break
+                except queue.Empty:
+                    pass
+
+                if batch:
+                    conn.executemany(
+                        _INSERT,
+                        [_entry_to_row(sid, e) for sid, e in batch],
+                    )
+                    conn.commit()
+        finally:
+            conn.close()
+
+
+class MessageStore:
+    """
+    Per-session message store backed by SQLite via a shared MessageWriter.
+
+    Drop-in replacement for MessageLog: same add/get_entries/clear/__len__ API.
+    """
+
+    def __init__(self, session_id: str, writer: MessageWriter):
+        self._session_id = session_id
+        self._writer = writer
+        self._db_path = writer._db_path
+
+    # ------------------------------------------------------------------
+    # Write
+    # ------------------------------------------------------------------
+
+    def add(self, direction: str, admin: bool, raw: str) -> LogEntry:
+        """Parse message, enqueue DB write, return LogEntry immediately."""
+        fields = parse_raw(raw)
+        mt = get_msg_type(fields)
+        entry = LogEntry(
+            direction=direction,
+            admin=admin,
+            raw=raw,
+            timestamp=datetime.now(tz=timezone.utc),
+            seq_num=get_seq_num(fields),
+            msg_type=mt,
+            msg_type_name=msg_type_name(mt) if mt else "Unknown",
+            fields=fields,
+        )
+        self._writer.enqueue(self._session_id, entry)
+        return entry
+
+    def clear(self) -> None:
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute(
+                "DELETE FROM messages WHERE session_id = ?", (self._session_id,)
+            )
+
+    # ------------------------------------------------------------------
+    # Read
+    # ------------------------------------------------------------------
+
+    def get_entries(
+        self,
+        *,
+        direction: Optional[str] = None,
+        admin: Optional[bool] = None,
+        msg_type: Optional[str] = None,
+        limit: Optional[int] = None,
+        before_id: Optional[int] = None,   # cursor: return entries with id < before_id
+    ) -> list[LogEntry]:
+        """Return entries matching filters, ordered oldest-first.
+
+        When before_id is set, only rows with id < before_id are considered
+        (cursor-based pagination for "Load older" queries).
+        When limit is set, the last N qualifying rows are returned.
+        """
+        where = ["session_id = ?"]
+        params: list = [self._session_id]
+
+        if direction is not None:
+            where.append("direction = ?")
+            params.append(direction)
+        if admin is not None:
+            where.append("admin = ?")
+            params.append(1 if admin else 0)
+        if msg_type is not None:
+            where.append("msg_type = ?")
+            params.append(msg_type)
+        if before_id is not None:
+            where.append("id < ?")
+            params.append(before_id)
+
+        where_sql = " AND ".join(where)
+        select = (
+            "SELECT id, session_id, ts, direction, admin, seq_num, "
+            f"msg_type, msg_type_name, raw, fields_json FROM messages WHERE {where_sql}"
+        )
+
+        if limit is not None:
+            # Fetch last N qualifying rows in ascending order
+            sql = f"SELECT * FROM ({select} ORDER BY id DESC LIMIT ?) ORDER BY id ASC"
+            params.append(limit)
+        else:
+            sql = f"{select} ORDER BY id ASC"
+
+        with sqlite3.connect(self._db_path) as conn:
+            rows = conn.execute(sql, params).fetchall()
+
+        return [_row_to_entry(r) for r in rows]
+
+    def __len__(self) -> int:
+        with sqlite3.connect(self._db_path) as conn:
+            row = conn.execute(
+                "SELECT COUNT(*) FROM messages WHERE session_id = ?",
+                (self._session_id,),
+            ).fetchone()
+        return row[0] if row else 0

fixture/core/models.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Optional, TYPE_CHECKING
+import uuid
+
+if TYPE_CHECKING:
+    from .message_log import MessageLog
+
+
+class ConnectionType(Enum):
+    INITIATOR = "initiator"
+    ACCEPTOR = "acceptor"
+
+
+class SessionRole(Enum):
+    CLIENT = "client"    # sends orders
+    VENUE = "venue"      # receives orders, sends fills
+    GENERIC = "generic"  # post-trade, IOIs, etc.
+
+
+class SessionStatus(Enum):
+    STOPPED = auto()
+    CONNECTING = auto()   # initiator: between start() and logon
+    LISTENING = auto()    # acceptor: socket bound, no client yet
+    LOGGED_ON = auto()
+    LOGGED_OUT = auto()   # engine running but no active logon
+    ERROR = auto()
+
+
+@dataclass
+class SessionConfig:
+    # Identity
+    session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    display_name: str = ""
+
+    # FIX identity
+    begin_string: str = "FIX.4.2"   # "FIX.4.2" or "FIX.4.4"
+    sender_comp_id: str = ""
+    target_comp_id: str = ""
+
+    # Connection
+    connection_type: ConnectionType = ConnectionType.INITIATOR
+    host: str = "localhost"          # ignored for acceptor
+    port: int = 9876
+    heartbeat_interval: int = 30
+    reconnect_interval: int = 10     # initiator only
+
+    # Data dictionary (None = use bundled spec)
+    data_dictionary_path: Optional[str] = None
+
+    # Store / log paths (use per-session subdirs to avoid collisions)
+    file_store_path: str = "./store"
+    file_log_path: str = "./log"
+    use_file_log: bool = True
+
+    # Session schedule ("00:00:00" for both = always active)
+    start_time: str = "00:00:00"
+    end_time: str = "00:00:00"
+
+    # Sequence number reset behaviour
+    reset_on_logon: bool = True
+    reset_on_logout: bool = False
+
+    # Role / intent
+    session_role: SessionRole = SessionRole.GENERIC
+
+    # Venue auto-responses (only active when session_role == VENUE)
+    auto_ack: bool = False
+    auto_ack_delay_ms: int = 100
+    auto_fill: bool = False
+    auto_fill_delay_ms: int = 500
+
+    # Ownership (empty string = legacy/unowned)
+    owner_uid: str = ""
+
+
+@dataclass
+class SessionState:
+    config: SessionConfig
+    status: SessionStatus = SessionStatus.STOPPED
+    error_message: str = ""
+    last_sent_seq_num: int = 0
+    last_recv_seq_num: int = 0
+    message_log: Optional[MessageLog] = field(default=None)  # set by Session on init
+    status_changed_at: Optional[float] = None  # time.time() of last status transition
+    last_heartbeat_at: Optional[float] = None  # time.time() of last inbound Heartbeat

fixture/core/scenario_runner.py

@@ -0,0 +1,331 @@
+"""
+Executes scenarios in background threads and emits progress via a broadcast callback.
+
+Each scenario run is a daemon thread that steps through send/expect/delay steps,
+subscribing to SessionManager events for 'expect' steps and using threading.Event
+for timeout and abort signalling.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Callable, Optional
+
+from fixcore.message.message import Message
+
+from .events import SessionEvent, EventType
+from .fix_parser import parse_raw
+from .session_manager import SessionManager
+from .template_store import TemplateStore
+
+SKIP_TAGS = {8, 9, 10, 34, 35, 49, 52, 56}
+
+
+def _now_ms() -> float:
+    return time.monotonic() * 1000
+
+
+def _utc_ts() -> str:
+    return datetime.now(timezone.utc).strftime("%Y%m%d-%H:%M:%S")
+
+
+def _parse_raw_multivalue(raw: str) -> dict[str, list[str]]:
+    """
+    Parse a raw FIX string preserving all occurrences of repeated tags.
+    Returns {tag_str: [val, val, ...]} so repeating group fields are not lost.
+    """
+    result: dict[str, list[str]] = {}
+    normalised = raw.replace("|", "\x01") if "\x01" not in raw else raw
+    for pair in normalised.split("\x01"):
+        if "=" in pair:
+            tag, _, val = pair.partition("=")
+            tag = tag.strip()
+            if tag:
+                result.setdefault(tag, []).append(val)
+    return result
+
+
+def _check_assertion(fields: dict[str, list[str]], assertion: dict) -> bool:
+    """
+    Return True if the message fields satisfy the assertion.
+    fields is a multi-value dict: {tag_str: [val, val, ...]}.
+    For eq/ne, the assertion passes if ANY occurrence satisfies it.
+    """
+    tag = str(assertion.get("tag", ""))
+    op = assertion.get("op", "eq")
+    expected = str(assertion.get("value", ""))
+
+    values = fields.get(tag)  # list[str] or None
+
+    if op == "exists":
+        return values is not None
+    if op == "absent":
+        return values is None
+    if values is None:
+        return False
+    if op == "eq":
+        return expected in values
+    if op == "ne":
+        return all(v != expected for v in values)
+    return False
+
+
+class ScenarioRunner:
+    def __init__(
+        self,
+        session_manager: SessionManager,
+        template_store: TemplateStore,
+        broadcast_fn: Callable[[str, dict], None],
+    ):
+        self._sm = session_manager
+        self._ts = template_store
+        self._broadcast = broadcast_fn
+        self._abort_flags: dict[str, threading.Event] = {}
+        self._lock = threading.Lock()
+
+    def run(self, scenario: dict, uid: str) -> str:
+        """Start scenario execution in a daemon thread. Returns run_id."""
+        run_id = str(uuid.uuid4())
+        abort_flag = threading.Event()
+        with self._lock:
+            self._abort_flags[run_id] = abort_flag
+        t = threading.Thread(
+            target=self._execute,
+            args=(scenario, uid, run_id, abort_flag),
+            daemon=True,
+        )
+        t.start()
+        return run_id
+
+    def abort(self, run_id: str) -> None:
+        """Signal abort for a running scenario."""
+        with self._lock:
+            flag = self._abort_flags.get(run_id)
+        if flag:
+            flag.set()
+
+    def _execute(self, scenario: dict, uid: str, run_id: str, abort_flag: threading.Event) -> None:
+        scenario_id = scenario.get("id", "")
+        scenario_name = scenario.get("name", "")
+        steps = scenario.get("steps", [])
+        step_count = len(steps)
+
+        self._broadcast(uid, {
+            "type": "scenario_start",
+            "run_id": run_id,
+            "scenario_id": scenario_id,
+            "scenario_name": scenario_name,
+            "step_count": step_count,
+        })
+
+        run_start = _now_ms()
+        overall_status = "pass"
+
+        for idx, step in enumerate(steps):
+            if abort_flag.is_set():
+                overall_status = "aborted"
+                break
+
+            label = step.get("label", f"Step {idx + 1}")
+            step_type = step.get("type", "")
+
+            self._broadcast(uid, {
+                "type": "scenario_step",
+                "run_id": run_id,
+                "step_index": idx,
+                "label": label,
+                "status": "running",
+                "error": None,
+                "duration_ms": 0,
+            })
+
+            step_start = _now_ms()
+            error: Optional[str] = None
+            status = "pass"
+
+            try:
+                if step_type == "send":
+                    error = self._run_send(step, scenario.get("session_id", ""), uid)
+                elif step_type == "expect":
+                    error = self._run_expect(step, scenario.get("session_id", ""), abort_flag)
+                elif step_type == "delay":
+                    self._run_delay(step, abort_flag)
+                    if abort_flag.is_set():
+                        overall_status = "aborted"
+                        status = "fail"
+                        error = "Aborted"
+            except Exception as exc:
+                error = str(exc)
+
+            if error is not None:
+                status = "fail"
+
+            duration_ms = int(_now_ms() - step_start)
+
+            self._broadcast(uid, {
+                "type": "scenario_step",
+                "run_id": run_id,
+                "step_index": idx,
+                "label": label,
+                "status": status,
+                "error": error,
+                "duration_ms": duration_ms,
+            })
+
+            if status == "fail":
+                if overall_status != "aborted":
+                    overall_status = "fail"
+                break
+
+        if abort_flag.is_set() and overall_status == "pass":
+            overall_status = "aborted"
+
+        total_ms = int(_now_ms() - run_start)
+
+        self._broadcast(uid, {
+            "type": "scenario_complete",
+            "run_id": run_id,
+            "status": overall_status,
+            "duration_ms": total_ms,
+        })
+
+        with self._lock:
+            self._abort_flags.pop(run_id, None)
+
+    def _run_send(self, step: dict, session_id: str, uid: str) -> Optional[str]:
+        """Build and send a FIX message. Returns error string or None.
+
+        If step contains a 'raw' field it is used directly, which preserves
+        repeating group structure (the raw string is sent via parse_raw +
+        setField, same code path as the REST send endpoint).
+        Otherwise, template + step fields are merged into a flat dict.
+        """
+        fields: dict[int, str] = {}
+
+        raw_field = step.get("raw", "").strip()
+        if raw_field:
+            # Use raw FIX string — this preserves repeated tags better than
+            # the flat fields dict.  parse_raw still flattens (last-value wins)
+            # for each tag, so groups with truly identical tag numbers across
+            # instances will lose earlier values, but the raw string is sent
+            # exactly as provided to QuickFIX which handles the wire format.
+            normalised = raw_field.replace("|", "\x01") if "\x01" not in raw_field else raw_field
+            fields = parse_raw(normalised)
+        else:
+            # Load template fields first
+            template_id = step.get("template_id")
+            if template_id:
+                tpl = self._ts.get_template(uid, template_id)
+                if tpl is None:
+                    return f"Template {template_id!r} not found"
+                for tag_str, value in tpl.get("fields", {}).items():
+                    try:
+                        fields[int(tag_str)] = value
+                    except ValueError:
+                        pass
+
+            # Step-level field overrides
+            for tag_str, value in step.get("fields", {}).items():
+                try:
+                    fields[int(tag_str)] = value
+                except ValueError:
+                    pass
+
+        msg_type = fields.get(35)
+        if not msg_type:
+            return "Missing MsgType (tag 35) in send step"
+
+        # Inject TransactTime if not present
+        if 60 not in fields:
+            fields[60] = _utc_ts()
+
+        msg = Message()
+        msg.header.set(35, msg_type)
+        for tag, value in fields.items():
+            if tag not in SKIP_TAGS:
+                try:
+                    msg.set_field(int(tag), str(value))
+                except Exception:
+                    pass
+
+        import asyncio
+        loop = asyncio.get_event_loop()
+        ok = asyncio.run_coroutine_threadsafe(
+            self._sm.send_message(session_id, msg), loop
+        ).result(timeout=5)
+        if not ok:
+            return "Session not logged on"
+        return None
+
+    def _run_expect(
+        self, step: dict, session_id: str, abort_flag: threading.Event
+    ) -> Optional[str]:
+        """Wait for an inbound message matching msg_type and assertions. Returns error or None."""
+        expected_msg_type = step.get("msg_type", "")
+        timeout_ms = step.get("timeout_ms", 5000)
+        assertions: list[dict] = step.get("assertions", [])
+
+        matched = threading.Event()
+        match_error: list[Optional[str]] = [None]  # mutable container
+
+        def handler(event: SessionEvent) -> None:
+            if matched.is_set() or abort_flag.is_set():
+                return
+            if event.event_type != EventType.MESSAGE_RECEIVED:
+                return
+            if event.session_id != session_id:
+                return
+            payload = event.payload
+            if expected_msg_type and payload.get("msg_type", "") != expected_msg_type:
+                return
+            # Use multi-value parser on the raw string so repeating group fields
+            # are not lost.  Falls back to the flat fields dict if raw is absent.
+            raw = payload.get("raw", "")
+            if raw:
+                fields = _parse_raw_multivalue(raw)
+            else:
+                fields = {str(k): [v] for k, v in payload.get("fields", {}).items()}
+            for assertion in assertions:
+                if not _check_assertion(fields, assertion):
+                    tag = assertion.get("tag", "?")
+                    op = assertion.get("op", "?")
+                    value = assertion.get("value", "")
+                    actual_list = fields.get(str(tag))
+                    actual_repr = repr(actual_list[0]) if actual_list and len(actual_list) == 1 else repr(actual_list) if actual_list else "<absent>"
+                    match_error[0] = (
+                        f"Assertion failed: tag {tag} {op} {value!r} (actual: {actual_repr})"
+                    )
+                    matched.set()
+                    return
+            matched.set()
+
+        self._sm.subscribe(handler)
+        try:
+            deadline = timeout_ms / 1000.0
+            poll_interval = 0.05
+            elapsed = 0.0
+            while elapsed < deadline:
+                if matched.is_set() or abort_flag.is_set():
+                    break
+                time.sleep(poll_interval)
+                elapsed += poll_interval
+        finally:
+            self._sm.unsubscribe(handler)
+
+        if abort_flag.is_set():
+            return "Aborted"
+        if not matched.is_set():
+            desc = expected_msg_type or "any"
+            return f"Timeout waiting for message type {desc!r} after {timeout_ms}ms"
+        return match_error[0]  # None = pass, non-None = assertion failure message
+
+    def _run_delay(self, step: dict, abort_flag: threading.Event) -> None:
+        delay_ms = step.get("delay_ms", 0)
+        remaining = delay_ms / 1000.0
+        interval = 0.05
+        while remaining > 0 and not abort_flag.is_set():
+            time.sleep(min(interval, remaining))
+            remaining -= interval