fixtureqa 0.1.8__tar.gz → 0.2.0__tar.gz

{fixtureqa-0.1.8/fixtureqa.egg-info → fixtureqa-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fixtureqa
-Version: 0.1.8
+Version: 0.2.0
 Summary: FIXture — FIX Protocol Testing Tool
 Requires-Python: >=3.10
 License-File: LICENSE

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/api/app.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 from contextlib import asynccontextmanager
 
@@ -19,6 +20,13 @@ from .routers import sessions, messages, ws, auth, setup, admin, fix_spec, templ
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    # Capture the running event loop so worker threads (scenarios, and later
+    # the perf injector) can submit coroutines back to it via
+    # run_coroutine_threadsafe. get_event_loop() does not work from a worker
+    # thread on Python 3.12.
+    loop = asyncio.get_running_loop()
+    app.state.loop = loop
+    app.state.scenario_runner.set_loop(loop)
     # Register event bridge with SessionManager
     sm: SessionManager = app.state.session_manager
     cm: ConnectionManager = app.state.connection_manager
@@ -50,12 +58,19 @@ def create_app(
         lifespan=lifespan,
     )
 
+    # Resolve a stable JWT signing key (persists under data_dir if no env var)
+    from ..core.auth import init_secret_key
+    init_secret_key(data_dir)
+
     # Store singletons on app state (injected via Depends)
     app.state.session_manager = session_manager
     app.state.connection_manager = ConnectionManager()
     user_store = UserStore(db_path)
     app.state.user_store = user_store
-    app.state.housekeeping = HousekeepingService(msg_db_path, log_dir, user_store)
+    app.state.housekeeping = HousekeepingService(
+        msg_db_path, log_dir, user_store,
+        message_writer=session_manager.message_writer,
+    )
     app.state.template_store = TemplateStore(data_dir)
     app.state.scenario_store = ScenarioStore(data_dir)
     app.state.custom_tag_store = CustomTagStore(data_dir)

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/api/connection_manager.py

@@ -73,14 +73,27 @@ class ConnectionManager:
 
     def __init__(self):
         self._clients: dict[str, dict[WebSocket, _ConnInfo]] = {}
+        # Hold strong references to in-flight broadcast tasks. asyncio only keeps
+        # a weak reference to tasks, so a fire-and-forget create_task() can be
+        # garbage-collected mid-execution, silently dropping a broadcast.
+        self._tasks: set[asyncio.Task] = set()
+
+    def _spawn(self, coro) -> None:
+        """Schedule a broadcast coroutine, retaining a reference until it finishes."""
+        task = asyncio.create_task(coro)
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
 
     # ------------------------------------------------------------------
     # Called from async context (WS handler)
     # ------------------------------------------------------------------
 
     async def connect(self, websocket: WebSocket, session_id: str = "*",
-                      uid: str = "", is_admin: bool = False) -> None:
-        await websocket.accept()
+                      uid: str = "", is_admin: bool = False,
+                      subprotocol: str | None = None) -> None:
+        # Echo the negotiated subprotocol ("bearer") when the token came in via
+        # the Sec-WebSocket-Protocol header, so the handshake completes cleanly.
+        await websocket.accept(subprotocol=subprotocol)
         self._clients.setdefault(session_id, {})[websocket] = _ConnInfo(uid, is_admin)
 
     def disconnect(self, websocket: WebSocket, session_id: str = "*") -> None:
@@ -94,11 +107,11 @@ class ConnectionManager:
     def sync_event_handler(self, event: SessionEvent) -> None:
         """Registered with SessionManager.subscribe(). Called on the event loop."""
         payload = _serialize_event(event)
-        asyncio.create_task(self._broadcast(event.session_id, payload, event.owner_uid))
+        self._spawn(self._broadcast(event.session_id, payload, event.owner_uid))
 
     def broadcast_scenario_event(self, uid: str, data: dict) -> None:
         """Schedule delivery to all WS connections for uid."""
-        asyncio.create_task(self._broadcast_to_user(uid, data))
+        self._spawn(self._broadcast_to_user(uid, data))
 
     # ------------------------------------------------------------------
     # Internal async broadcast

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/api/routers/ws.py

@@ -40,19 +40,35 @@ async def _send_snapshot(websocket: WebSocket, sm: SessionManager, uid: str, is_
         })
 
 
-def _authenticate_ws(websocket: WebSocket, token: Optional[str]) -> tuple[str, bool]:
-    """Validate JWT token from query param. Returns (uid, is_admin) or closes with 403."""
+def _token_from_ws(websocket: WebSocket, query_token: Optional[str]) -> tuple[Optional[str], Optional[str]]:
+    """Resolve the JWT, preferring the Sec-WebSocket-Protocol header so the
+    token never appears in the URL (and thus not in access/proxy logs or
+    browser history). The browser sends ["bearer", "<jwt>"]; we echo "bearer"
+    as the negotiated subprotocol. Falls back to the legacy query param.
+
+    Returns (token, subprotocol_to_echo).
+    """
+    raw = websocket.headers.get("sec-websocket-protocol", "")
+    parts = [p.strip() for p in raw.split(",") if p.strip()]
+    if len(parts) >= 2 and parts[0] == "bearer":
+        return parts[1], "bearer"
+    return query_token, None
+
+
+def _authenticate_ws(websocket: WebSocket, token: Optional[str]) -> tuple[Optional[str], bool, Optional[str]]:
+    """Validate the JWT. Returns (uid, is_admin, subprotocol) or (None, False, None)."""
+    token, subprotocol = _token_from_ws(websocket, token)
     if not token:
-        return None, False
+        return None, False, None
     try:
         payload = decode_token(token)
         uid = payload.get("sub", "")
         role = payload.get("role", "")
         if not uid:
-            return None, False
-        return uid, role == "platform_admin"
+            return None, False, None
+        return uid, role == "platform_admin", subprotocol
     except JWTError:
-        return None, False
+        return None, False, None
 
 
 @router.websocket("/ws")
@@ -63,7 +79,7 @@ async def ws_all_sessions(
     token: Optional[str] = Query(default=None),
 ):
     """Subscribe to events from all sessions."""
-    uid, is_admin = _authenticate_ws(websocket, token)
+    uid, is_admin, subprotocol = _authenticate_ws(websocket, token)
     if not uid:
         logger.warning("WS /ws: auth failed (no token or invalid JWT)")
         await websocket.close(code=4003, reason="Unauthorized")
@@ -78,7 +94,8 @@ async def ws_all_sessions(
         return
 
     logger.info("WS /ws: client connected uid=%s is_admin=%s", uid, is_admin)
-    await conn_mgr.connect(websocket, session_id="*", uid=uid, is_admin=is_admin)
+    await conn_mgr.connect(websocket, session_id="*", uid=uid, is_admin=is_admin,
+                           subprotocol=subprotocol)
     try:
         await _send_snapshot(websocket, sm, uid, is_admin)
         while True:
@@ -103,7 +120,7 @@ async def ws_single_session(
     token: Optional[str] = Query(default=None),
 ):
     """Subscribe to events from a single session."""
-    uid, is_admin = _authenticate_ws(websocket, token)
+    uid, is_admin, subprotocol = _authenticate_ws(websocket, token)
     if not uid:
         logger.warning("WS /ws/%s: auth failed (no token or invalid JWT)", session_id)
         await websocket.close(code=4003, reason="Unauthorized")
@@ -125,7 +142,8 @@ async def ws_single_session(
         return
 
     logger.info("WS /ws/%s: client connected uid=%s is_admin=%s", session_id, uid, is_admin)
-    await conn_mgr.connect(websocket, session_id=session_id, uid=uid, is_admin=is_admin)
+    await conn_mgr.connect(websocket, session_id=session_id, uid=uid, is_admin=is_admin,
+                           subprotocol=subprotocol)
     try:
         await _send_snapshot(websocket, sm, uid, is_admin, session_id=session_id)
         while True:

fixtureqa-0.2.0/fixture/core/atomic_io.py

@@ -0,0 +1,38 @@
+"""
+Atomic JSON file writes shared by the JSON-backed stores.
+
+Each store previously wrote to a fixed "<path>.tmp" then os.replace()'d it.
+Two concurrent writers to the same path clobbered that single temp file,
+so the bytes promoted by os.replace could be a corrupt interleaving. This
+helper writes to a uniquely-named temp file in the same directory, fsyncs it,
+then renames over the target — atomic and collision-free.
+"""
+import json
+import os
+import tempfile
+
+
+def atomic_write_json(path: str, data, *, indent: int = 2) -> None:
+    """Atomically serialize `data` as JSON to `path`.
+
+    The temp file is created in the same directory as `path` (so os.replace is
+    a same-filesystem atomic rename) with a unique name (so concurrent writers
+    don't share a temp file). On any failure the temp file is removed.
+    """
+    directory = os.path.dirname(path) or "."
+    os.makedirs(directory, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(
+        prefix=os.path.basename(path) + ".", suffix=".tmp", dir=directory
+    )
+    try:
+        with os.fdopen(fd, "w") as f:
+            json.dump(data, f, indent=indent)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise

fixtureqa-0.2.0/fixture/core/auth.py

@@ -0,0 +1,105 @@
+"""
+Password hashing and JWT helpers.
+
+Secret key is read from the FIXTURE_SECRET_KEY environment variable.
+If absent a random key is generated for the current process (tokens
+won't survive a restart — acceptable for dev, not for production).
+"""
+import logging
+import os
+import secrets
+
+import bcrypt as _bcrypt
+
+from jose import JWTError, jwt
+
+from .user_store import User
+
+logger = logging.getLogger(__name__)
+
+_ALGORITHM = "HS256"
+_EXPIRY_SECONDS = 8 * 60 * 60  # 8 hours
+
+# Set from FIXTURE_SECRET_KEY at import; otherwise resolved by init_secret_key()
+# (persistent file) or, as a last resort, lazily generated by _get_secret().
+_SECRET_KEY: str = os.environ.get("FIXTURE_SECRET_KEY", "")
+
+
+def init_secret_key(data_dir: str) -> None:
+    """Resolve a stable signing key when FIXTURE_SECRET_KEY is not set.
+
+    Loads (or creates) a persistent key file under data_dir so tokens survive
+    restarts in dev. Production should still set FIXTURE_SECRET_KEY explicitly
+    — it is required for multi-process deployments, where each worker would
+    otherwise generate its own key and reject each other's tokens.
+    """
+    global _SECRET_KEY
+    if _SECRET_KEY:
+        return  # explicit env var wins
+
+    key_path = os.path.join(data_dir, ".secret_key")
+    try:
+        if os.path.isfile(key_path):
+            with open(key_path) as f:
+                _SECRET_KEY = f.read().strip()
+        if not _SECRET_KEY:
+            _SECRET_KEY = secrets.token_hex(32)
+            os.makedirs(data_dir, exist_ok=True)
+            # 0600 — the key must not be world-readable.
+            fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+            with os.fdopen(fd, "w") as f:
+                f.write(_SECRET_KEY)
+            logger.warning(
+                "FIXTURE_SECRET_KEY not set — generated a persistent key at %s. "
+                "Set FIXTURE_SECRET_KEY in production (required for multi-process).",
+                key_path,
+            )
+    except OSError as e:
+        logger.warning("Could not persist secret key (%s); falling back to ephemeral.", e)
+
+
+def _get_secret() -> str:
+    global _SECRET_KEY
+    if not _SECRET_KEY:
+        # Last resort: no env var and init_secret_key never ran / failed.
+        _SECRET_KEY = secrets.token_hex(32)
+        logger.warning(
+            "FIXTURE_SECRET_KEY not set — using a random ephemeral key. "
+            "Tokens will be invalidated on restart."
+        )
+    return _SECRET_KEY
+
+
+def _encode(plain: str) -> bytes:
+    # bcrypt hard-limits at 72 bytes — truncate to avoid ValueError in bcrypt 5.x
+    return plain.encode("utf-8")[:72]
+
+
+def hash_password(plain: str) -> str:
+    return _bcrypt.hashpw(_encode(plain), _bcrypt.gensalt()).decode("utf-8")
+
+
+def verify_password(plain: str, hashed: str) -> bool:
+    try:
+        return _bcrypt.checkpw(_encode(plain), hashed.encode("utf-8"))
+    except Exception:
+        return False
+
+
+def create_token(user: User) -> str:
+    import time
+    payload = {
+        "sub": user.uid,
+        "username": user.username,
+        "role": user.role,
+        "exp": int(time.time()) + _EXPIRY_SECONDS,
+    }
+    return jwt.encode(payload, _get_secret(), algorithm=_ALGORITHM)
+
+
+def decode_token(token: str) -> dict:
+    """
+    Decode and validate a JWT. Raises jose.JWTError on invalid/expired token.
+    Returns the raw payload dict.
+    """
+    return jwt.decode(token, _get_secret(), algorithms=[_ALGORITHM])

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/core/config_store.py

@@ -1,7 +1,8 @@
 import json
 import os
-from typing import List
+from typing import Dict, List
 
+from .atomic_io import atomic_write_json
 from .models import SessionConfig, ConnectionType, SessionRole
 
 
@@ -28,12 +29,28 @@ class ConfigStore:
     def save_for_user(self, uid: str, configs: List[SessionConfig]) -> None:
         """Atomically save one user's sessions to data_dir/<uid>/sessions.json."""
         user_dir = os.path.join(self._data_dir, uid)
-        os.makedirs(user_dir, exist_ok=True)
         path = os.path.join(user_dir, "sessions.json")
-        tmp = path + ".tmp"
-        with open(tmp, "w") as f:
-            json.dump([_config_to_dict(c) for c in configs], f, indent=2)
-        os.replace(tmp, path)
+        atomic_write_json(path, [_config_to_dict(c) for c in configs])
+
+    def save_all(self, configs_by_user: Dict[str, List[SessionConfig]]) -> None:
+        """Persist every user's sessions and clear files for users with none left.
+
+        Reconciles against on-disk state: a user whose last session was removed
+        no longer appears in configs_by_user, so their sessions.json is emptied.
+        Without this, a deleted session reappears on the next load_all().
+        """
+        for uid, configs in configs_by_user.items():
+            self.save_for_user(uid, configs)
+
+        if not os.path.isdir(self._data_dir):
+            return
+        for uid in os.listdir(self._data_dir):
+            if uid in configs_by_user:
+                continue
+            # Only rewrite users that already have a persisted file; skip
+            # non-user entries in data_dir (messages.db, logs, etc.).
+            if os.path.isfile(os.path.join(self._data_dir, uid, "sessions.json")):
+                self.save_for_user(uid, [])
 
 
 def _load_file(path: str) -> List[SessionConfig]:

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/core/custom_tag_store.py

@@ -4,6 +4,8 @@ import json
 import os
 from threading import Lock
 
+from .atomic_io import atomic_write_json
+
 
 class CustomTagStore:
     def __init__(self, data_dir: str) -> None:
@@ -21,12 +23,7 @@ class CustomTagStore:
             return {}
 
     def _save(self, uid: str, tags: dict[int, dict]) -> None:
-        path = self._path(uid)
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        tmp = path + ".tmp"
-        with open(tmp, "w") as f:
-            json.dump({str(k): v for k, v in tags.items()}, f, indent=2)
-        os.replace(tmp, path)
+        atomic_write_json(self._path(uid), {str(k): v for k, v in tags.items()})
 
     def list_tags(self, uid: str) -> list[dict]:
         with self._lock:

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/core/fix_application.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 
 from fixcore.application import Application
 from fixcore.message.message import Message
@@ -44,11 +45,21 @@ class FixApplication(Application):
             pass
 
     def from_admin(self, message: Message, session_id: SessionID) -> None:
-        self._emit(EventType.MESSAGE_RECEIVED, {"raw": message.encode().decode("ascii", errors="replace"), "admin": True})
+        recv_perf_ns = time.perf_counter_ns()
+        self._emit(EventType.MESSAGE_RECEIVED, {
+            "raw": message.encode().decode("ascii", errors="replace"),
+            "admin": True,
+            "recv_perf_ns": recv_perf_ns,
+        })
 
     def from_app(self, message: Message, session_id: SessionID) -> None:
+        recv_perf_ns = time.perf_counter_ns()
         try:
-            self._emit(EventType.MESSAGE_RECEIVED, {"raw": message.encode().decode("ascii", errors="replace"), "admin": False})
+            self._emit(EventType.MESSAGE_RECEIVED, {
+                "raw": message.encode().decode("ascii", errors="replace"),
+                "admin": False,
+                "recv_perf_ns": recv_perf_ns,
+            })
         except Exception:
             pass
 

{fixtureqa-0.1.8 → fixtureqa-0.2.0}/fixture/core/housekeeping.py

@@ -28,10 +28,15 @@ class HousekeepingService:
     admin UI take effect on the next scheduled run without a restart.
     """
 
-    def __init__(self, msg_db_path: str, log_dir: str, user_store: "UserStore"):
+    def __init__(self, msg_db_path: str, log_dir: str, user_store: "UserStore",
+                 message_writer=None):
         self._msg_db_path = msg_db_path
         self._log_dir = log_dir
         self._user_store = user_store
+        # When present, purges are funneled through the single message writer so
+        # messages.db only ever has one writer. Falls back to its own connection
+        # when message logging is disabled (no writer).
+        self._message_writer = message_writer
 
     def start(self) -> None:
         """Start the background daemon thread."""
@@ -80,9 +85,13 @@ class HousekeepingService:
     def _purge_messages(self, retention_days: int) -> int:
         if retention_days == 0:
             return 0
+        cutoff = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).isoformat()
+        if self._message_writer is not None:
+            # Single-writer path: the writer thread owns the only write connection.
+            return self._message_writer.purge(cutoff)
+        # Fallback: message logging disabled — no central writer, so no contention.
         if not os.path.isfile(self._msg_db_path):
             return 0
-        cutoff = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).isoformat()
         with sqlite3.connect(self._msg_db_path, timeout=30) as conn:
             cur = conn.execute("DELETE FROM messages WHERE ts < ?", (cutoff,))
         return cur.rowcount

fixtureqa-0.2.0/fixture/core/inbound.py

@@ -0,0 +1,104 @@
+"""
+Per-session inbound message subscriptions.
+
+This is the stable interface a high-rate consumer (e.g. the perf correlator)
+builds against. A subscriber gets a *bounded* asyncio.Queue fed from the event
+loop. The producer side (offer) is O(1) and never blocks the loop; when the
+consumer falls behind and the buffer fills, the oldest message is evicted and a
+`dropped` counter is incremented so loss is visible and measurable.
+
+Threading: offer() and get() are both expected to run on the FastAPI/uvicorn
+event loop (fixcore callbacks dispatch there; perf consumers are loop tasks).
+asyncio.Queue is not thread-safe — do not call these from a worker thread.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .session_manager import SessionManager
+
+DEFAULT_MAXSIZE = 10_000
+
+
+@dataclass
+class InboundMessage:
+    """A single inbound (received) FIX message handed to a subscriber."""
+    session_id: str
+    admin: bool             # True = session-level (Logon, Heartbeat, ...)
+    raw: str                # raw FIX string (SOH-delimited)
+    msg_type: str           # MsgType (tag 35)
+    seq_num: int            # MsgSeqNum (tag 34); 0 if absent
+    fields: dict            # {tag_int: value_str}
+    recv_perf_ns: int       # time.perf_counter_ns() at fixcore delivery
+
+
+class SessionSubscription:
+    """
+    Bounded, drop-oldest feed of inbound messages for one session.
+
+    Producer (event loop) calls offer(); consumer (a perf correlator task)
+    awaits get(). Close via close() or use as a context manager to deregister.
+    """
+
+    def __init__(self, manager: "SessionManager", session_id: str,
+                 maxsize: int = DEFAULT_MAXSIZE):
+        self._manager = manager
+        self.session_id = session_id
+        self._queue: "asyncio.Queue[InboundMessage]" = asyncio.Queue(maxsize=maxsize)
+        self.dropped = 0
+        self._closed = False
+
+    # ------------------------------------------------------------------
+    # Producer side (event loop)
+    # ------------------------------------------------------------------
+
+    def offer(self, msg: InboundMessage) -> None:
+        """Enqueue a message. O(1), never blocks. Evicts oldest when full."""
+        try:
+            self._queue.put_nowait(msg)
+        except asyncio.QueueFull:
+            try:
+                self._queue.get_nowait()  # evict oldest
+            except asyncio.QueueEmpty:
+                pass
+            self.dropped += 1
+            try:
+                self._queue.put_nowait(msg)
+            except asyncio.QueueFull:
+                # Should not happen on a single-threaded loop, but stay safe.
+                self.dropped += 1
+
+    # ------------------------------------------------------------------
+    # Consumer side
+    # ------------------------------------------------------------------
+
+    async def get(self) -> InboundMessage:
+        """Await the next inbound message."""
+        return await self._queue.get()
+
+    def get_nowait(self) -> InboundMessage:
+        """Return the next message or raise asyncio.QueueEmpty."""
+        return self._queue.get_nowait()
+
+    def qsize(self) -> int:
+        return self._queue.qsize()
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def close(self) -> None:
+        """Deregister from the manager. Idempotent."""
+        if not self._closed:
+            self._closed = True
+            self._manager._remove_subscription(self)
+
+    def __enter__(self) -> "SessionSubscription":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()