fixtureqa 0.4.3__tar.gz → 0.4.5__tar.gz

{fixtureqa-0.4.3/fixtureqa.egg-info → fixtureqa-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fixtureqa
-Version: 0.4.3
+Version: 0.4.5
 Summary: FIXture — FIX Protocol Testing Tool
 Requires-Python: >=3.10
 License-File: LICENSE

{fixtureqa-0.4.3 → fixtureqa-0.4.5}/fixture/api/app.py

@@ -90,7 +90,8 @@ def create_app(
     app.state.spec_overlay_store = SpecOverlayStore(data_dir)
     perf_writer = PerfWriter(os.path.join(data_dir, "perf.db"))
     app.state.perf_writer = perf_writer
-    app.state.perf_registry = RunRegistry(perf_writer)
+    app.state.perf_registry = RunRegistry(
+        perf_writer, execs_dir=os.path.join(data_dir, "perf_execs"))
     # Runs left non-terminal in perf.db were interrupted by a prior restart —
     # mark them errored so History doesn't show zombie 'running' rows.
     app.state.perf_registry.reconcile_interrupted()

{fixtureqa-0.4.3 → fixtureqa-0.4.5}/fixture/api/routers/perf.py

@@ -12,7 +12,7 @@ import zipfile
 from typing import Annotated, Optional
 
 from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
-from fastapi.responses import Response, StreamingResponse
+from fastapi.responses import FileResponse, Response, StreamingResponse
 
 from ...core.models import SessionStatus
 from ...core.perf_models import RunConfig, RunStatus
@@ -136,6 +136,11 @@ def run_scenarios(run_id: str, reg: REG, pw: PW, _admin: Admin,
     return pw.read_scenarios(run_id, limit, offset)
 
 
+def _run_is_live(reg: RunRegistry, run_id: str) -> bool:
+    run = reg.get(run_id)
+    return run is not None and run.status not in _TERMINAL
+
+
 def _csv_stream(pw: PerfWriter, table: str, cols: tuple, run_id: str):
     buf = io.StringIO()
     w = csv.writer(buf)
@@ -172,6 +177,19 @@ def run_export(run_id: str, reg: REG, pw: PW, _admin: Admin,
             _csv_stream(pw, "perf_scenarios", pw.scenario_columns(), run_id),
             media_type="text/csv",
             headers={"Content-Disposition": f'attachment; filename="{run_id}_scenarios.csv"'})
+    if kind == "execs":
+        # CSV-direct: the run streams per-exec rows straight into this csv.gz; the
+        # gzip footer is only written when the run finishes, so a live file would
+        # download truncated.
+        if _run_is_live(reg, run_id):
+            raise HTTPException(status_code=409,
+                                detail="Run still active — the exec CSV is finalized when the run finishes")
+        path = reg.exec_csv_path(run_id)
+        if path is None:
+            raise HTTPException(status_code=404,
+                                detail="No exec recording for this run (enable record_execs before starting)")
+        return FileResponse(path, media_type="application/gzip",
+                            filename=f"{run_id}_execs.csv.gz")
     if kind == "both":
         buf = io.BytesIO()
         with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
@@ -179,9 +197,14 @@ def run_export(run_id: str, reg: REG, pw: PW, _admin: Admin,
                        _csv_text(pw, "perf_messages", pw.message_columns(), run_id))
             z.writestr(f"{run_id}_scenarios.csv",
                        _csv_text(pw, "perf_scenarios", pw.scenario_columns(), run_id))
+            execs_path = reg.exec_csv_path(run_id)
+            if execs_path is not None and not _run_is_live(reg, run_id):
+                # Already gzipped — store as-is rather than re-deflating.
+                z.write(execs_path, arcname=f"{run_id}_execs.csv.gz",
+                        compress_type=zipfile.ZIP_STORED)
         return Response(buf.getvalue(), media_type="application/zip",
                         headers={"Content-Disposition": f'attachment; filename="{run_id}.zip"'})
-    raise HTTPException(status_code=422, detail="kind must be messages|scenarios|both")
+    raise HTTPException(status_code=422, detail="kind must be messages|scenarios|execs|both")
 
 
 # --- saved configs --------------------------------------------------------

fixtureqa-0.4.5/fixture/core/exec_csv_writer.py

@@ -0,0 +1,81 @@
+"""
+Per-run gzipped CSV recorder for individual ExecutionReports (opt-in via
+test.record_execs).
+
+CSV-direct: there is no database staging — rows stream straight to
+data/perf_execs/<run_id>.csv.gz and export serves the finished file as-is.
+Same non-blocking producer contract as PerfWriter: the event loop only does a
+put_nowait (drop+count when the bounded queue is full); all formatting and
+gzip/disk work happens on one daemon writer thread, so recording cannot add
+latency to the measured inbound path.
+
+One instance per recording run; it owns its file exclusively for its lifetime.
+"""
+from __future__ import annotations
+
+import csv
+import gzip
+import logging
+import os
+import queue
+import threading
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MAX_QUEUE = 200_000
+
+EXEC_COLS = (
+    "run_id", "corr_id", "clordid", "exec_id",
+    "exec_type", "ord_status",                      # tags 150 / 39
+    "last_qty", "last_px", "cum_qty", "leaves_qty",  # tags 32 / 31 / 14 / 151
+    "exec_seq",      # 1-based report index within the order
+    "sent_at",       # wall clock of the order send (same for all the order's rows)
+    "recv_at",       # wall clock of this report's receive
+    "latency_us",    # client receive − order send (one monotonic clock — authoritative;
+                     # recv_at − sent_at is wall clock, for cross-referencing only)
+    "gap_us",        # since the previous report for this order; empty on the first
+)
+
+
+class ExecCsvWriter:
+    def __init__(self, path: str, max_queue: int = DEFAULT_MAX_QUEUE):
+        self.path = path
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        self._queue: queue.Queue = queue.Queue(maxsize=max_queue)
+        self.dropped = 0
+        self._thread = threading.Thread(target=self._run, name="perf-exec-csv", daemon=True)
+        self._thread.start()
+
+    # -- producer side (event loop; never blocks) -----------------------
+
+    def enqueue(self, row: tuple) -> None:
+        try:
+            self._queue.put_nowait(row)
+        except queue.Full:
+            self.dropped += 1
+            if self.dropped % 1000 == 1:
+                logger.warning("exec csv queue full — dropped %d row(s) so far", self.dropped)
+
+    def close(self) -> None:
+        """Drain, finalize the gzip stream and stop. Blocks on the thread join —
+        call off-loop (asyncio.to_thread)."""
+        try:
+            self._queue.put(None, timeout=5)
+        except queue.Full:
+            logger.error("exec csv queue full on close — file %s may be incomplete", self.path)
+        self._thread.join(timeout=30.0)
+
+    # -- writer thread ---------------------------------------------------
+
+    def _run(self) -> None:
+        try:
+            with gzip.open(self.path, "wt", newline="") as fh:
+                w = csv.writer(fh)
+                w.writerow(EXEC_COLS)
+                while True:
+                    item = self._queue.get()
+                    if item is None:
+                        return   # with-block flushes + writes the gzip footer
+                    w.writerow(item)
+        except Exception:
+            logger.exception("exec csv writer failed (%s)", self.path)

{fixtureqa-0.4.3 → fixtureqa-0.4.5}/fixture/core/perf_engine.py

@@ -21,6 +21,7 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import os
 import random
 import time
 import uuid
@@ -28,6 +29,7 @@ from typing import Optional
 
 from fixcore.message.message import Message
 
+from .exec_csv_writer import ExecCsvWriter
 from .fix_time import utc_timestamp
 from .perf_models import (
     RunConfig, RunStatus, LiveSnapshot, ClientLeg, VenueLeg, SnapshotErrors, LatencyStats,
@@ -80,11 +82,14 @@ def _to_float(v) -> Optional[float]:
 class PerfRun:
     def __init__(self, run_id: str, config: RunConfig, sm: SessionManager, writer: PerfWriter,
                  template_store=None, owner_uid: str = "",
-                 sender_comp_id: str = "", target_comp_id: str = ""):
+                 sender_comp_id: str = "", target_comp_id: str = "",
+                 execs_dir: Optional[str] = None):
         self.run_id = run_id
         self.config = config
         self._sm = sm
         self._writer = writer
+        self._execs_dir = execs_dir
+        self._exec_csv: Optional[ExecCsvWriter] = None
 
         self.status: str = "pending"
         self.started_at: Optional[float] = None
@@ -131,6 +136,16 @@ class PerfRun:
         self.status = "running"
         self.started_at = time.time()
         self._start_ns = time.perf_counter_ns()
+        if self.config.test.record_execs:
+            # Per-exec capture happens in the client-leg correlator; venue-only
+            # runs have nothing to record.
+            if not self._has_client:
+                self.warnings.append("record_execs has no effect in venue mode (no client leg)")
+            elif not self._execs_dir:
+                self.warnings.append("record_execs unavailable (no exec CSV directory configured)")
+            else:
+                self._exec_csv = ExecCsvWriter(
+                    os.path.join(self._execs_dir, f"{self.run_id}.csv.gz"))
         if self._has_client:
             self._client_sub = self._sm.subscribe_session(self.config.client_session_id)
         if self._has_venue:
@@ -192,6 +207,10 @@ class PerfRun:
             self._client_sub.close()
         if self._venue_sub:
             self._venue_sub.close()
+        if self._exec_csv is not None:
+            # Correlator is cancelled (no more enqueues); drain + gzip-finalize
+            # off-loop — close() joins the writer thread.
+            await asyncio.to_thread(self._exec_csv.close)
         self.status = "stopped" if self._stopped_by_user else "completed"
         self._build_snapshot()
         self._publish()
@@ -482,6 +501,20 @@ class PerfRun:
                 return
             entry["seen_exec"].add(exec_id)
 
+        if self._exec_csv is not None:
+            seq = entry["exec_seq"] = entry.get("exec_seq", 0) + 1
+            prev_ns = entry.get("last_recv_ns")
+            entry["last_recv_ns"] = now_ns
+            self._exec_csv.enqueue((
+                self.run_id, corr, entry.get("clordid"), exec_id,
+                f.get(T_EXECTYPE), f.get(T_ORDSTATUS),
+                _to_float(f.get(T_LASTQTY)), _to_float(f.get(T_LASTPX)),
+                _to_float(f.get(T_CUMQTY)), _to_float(f.get(T_LEAVESQTY)),
+                seq, entry.get("sent_at"), time.time(),
+                (now_ns - entry["sent_ns"]) / 1000.0,
+                ((now_ns - prev_ns) / 1000.0) if prev_ns is not None else None,
+            ))
+
         if not entry["responded"]:
             entry["responded"] = True
             entry["first_resp_at"] = time.time()
@@ -640,8 +673,9 @@ class RunRegistry:
     TICKET_TTL_S = 30.0
     _TERMINAL_STATES = ("completed", "stopped", "error")
 
-    def __init__(self, writer: PerfWriter):
+    def __init__(self, writer: PerfWriter, execs_dir: Optional[str] = None):
         self._writer = writer
+        self._execs_dir = execs_dir
         self._runs: dict[str, PerfRun] = {}
         self._tickets: dict[str, tuple[str, float]] = {}
 
@@ -649,11 +683,19 @@ class RunRegistry:
                owner_uid: str = "", sender_comp_id: str = "", target_comp_id: str = "") -> PerfRun:
         run_id = str(uuid.uuid4())
         run = PerfRun(run_id, config, sm, self._writer, template_store, owner_uid,
-                      sender_comp_id, target_comp_id)
+                      sender_comp_id, target_comp_id, execs_dir=self._execs_dir)
         self._runs[run_id] = run
         run.start()
         return run
 
+    def exec_csv_path(self, run_id: str) -> Optional[str]:
+        """Path to the run's per-exec csv.gz, or None if it never recorded.
+        The file is complete (gzip footer written) only once the run is terminal."""
+        if not self._execs_dir:
+            return None
+        path = os.path.join(self._execs_dir, f"{run_id}.csv.gz")
+        return path if os.path.exists(path) else None
+
     def get(self, run_id: str) -> Optional[PerfRun]:
         """The live PerfRun, or None if it's not in this process (e.g. historical).
         Used for actions that need the live object: stop, WS watcher, ticket."""

{fixtureqa-0.4.3 → fixtureqa-0.4.5}/fixture/core/perf_models.py

@@ -43,6 +43,10 @@ class TestConfig(BaseModel):
     max_pending: int = Field(default=100_000, gt=0)
     on_saturation: SaturationPolicy = "pause"
     output: str = "results"
+    # Record every inbound ExecutionReport (ack + each fill) to a per-run
+    # csv.gz for detailed latency analysis. Off by default: ~(1+fills_per_order)×
+    # the per-order row volume on disk.
+    record_execs: bool = False
 
 
 class OrderPayloadConfig(BaseModel):