fixtureqa 0.4.9__tar.gz → 0.4.11__tar.gz

{fixtureqa-0.4.9/fixtureqa.egg-info → fixtureqa-0.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fixtureqa
-Version: 0.4.9
+Version: 0.4.11
 Summary: FIXture — FIX Protocol Testing Tool
 Requires-Python: >=3.10
 License-File: LICENSE

{fixtureqa-0.4.9 → fixtureqa-0.4.11}/fixture/core/perf_engine.py

@@ -25,6 +25,7 @@ import os
 import random
 import time
 import uuid
+from collections import deque
 from typing import Optional
 
 from fixcore.message.message import Message
@@ -111,6 +112,13 @@ class PerfRun:
         self._amend_on = config.amend.enabled and self._has_client
         self._amend_tasks: set[asyncio.Task] = set()
 
+        # Venue order book: corr → resumable fill state. The dispatcher books
+        # orders; _fill_sender drains them round-robin (via _fill_runnable),
+        # pausing at predicted amend trigger points when amends are enabled.
+        self._venue_orders: dict[str, dict] = {}
+        self._fill_runnable: deque[str] = deque()
+        self._fill_wakeup = asyncio.Event()
+
         # PayloadFactory validates templates/tokens now — raises PerfConfigError
         # (surfaced as 422) before any task starts.
         self._factory = PayloadFactory(config, template_store, owner_uid,
@@ -180,7 +188,7 @@ class PerfRun:
         if self._has_client:
             tasks += [self._injector(), self._correlator(), self._timeout_sweeper()]
         if self._has_venue:
-            tasks += [self._fill_dispatcher()]
+            tasks += [self._fill_dispatcher(), self._fill_sender()]
         self._tasks = [asyncio.create_task(t) for t in tasks]
         dur = self.config.test.duration
         try:
@@ -218,6 +226,8 @@ class PerfRun:
                 self.stats.amends_lost += 1
             self._finalize(entry, "filled" if entry.get("completed") else "lost")
         self._pending.clear()
+        self._venue_orders.clear()
+        self._fill_runnable.clear()
         if self._client_sub:
             self._client_sub.close()
         if self._venue_sub:
@@ -245,8 +255,13 @@ class PerfRun:
         r = self.config.rate
         fill_rate = r.fills_per_window / (r.fill_window_ms / 1000.0)   # fills/sec
         quiet = max(0.3, (3.0 / fill_rate) if fill_rate > 0 else 0.3)
-        backlog = self._venue_sub.qsize() * max(1, self.config.payload.fill.fills_per_order)
-        cap = time.perf_counter() + max(30.0, (backlog / max(1.0, fill_rate)) * 2.0 + 10.0)
+        # Backlog = unread inbound orders (not yet booked) + booked orders'
+        # remaining scheduled fills. Held orders resume via grace at the
+        # latest (no new holds once injection stops), so add one grace.
+        backlog = (self._venue_sub.qsize() * max(1, self.config.payload.fill.fills_per_order)
+                   + sum(len(e["schedule"]) - e["idx"] for e in self._venue_orders.values()))
+        cap = time.perf_counter() + max(30.0, (backlog / max(1.0, fill_rate)) * 2.0
+                                        + self.config.amend.amend_wait_ms / 1000.0 + 10.0)
 
         last_sent = self.stats.fills_sent + self.stats.acks_sent
         last_activity = time.perf_counter()
@@ -260,10 +275,12 @@ class PerfRun:
             if cur_sent != last_sent:
                 last_sent = cur_sent
                 last_activity = now
-            # Done when the venue has drained its backlog, the client has received
-            # the fills, and no new fill/ack has gone out for a quiet window
-            # (covers the rate-limited last order still being filled).
+            # Done when the venue has drained its backlog (inbound queue AND
+            # order book), the client has received the fills, and no new
+            # fill/ack has gone out for a quiet window (covers the
+            # rate-limited last order still being filled).
             quiet_ok = (self._venue_sub.qsize() == 0 and self._client_sub.qsize() == 0
+                        and not self._venue_orders
                         and (now - last_activity) >= quiet)
             if quiet_ok:
                 break
@@ -396,6 +413,10 @@ class PerfRun:
     # ------------------------------------------------------------------
 
     async def _fill_dispatcher(self) -> None:
+        """Venue inbound router. Never blocks on fill pacing: orders are booked
+        with a resumable schedule for _fill_sender, so an inbound 35=G is
+        handled the moment it arrives instead of queueing behind the very
+        fills that triggered it."""
         cfg = self.config
         f = cfg.payload.fill
         sub = self._venue_sub
@@ -409,9 +430,9 @@ class PerfRun:
             self.stats.orders_received += 1
             of = inb.fields
             order_qty = _to_int(of.get(T_ORDERQTY), 0)
-            if of.get(cfg.correlation_tag) is None or order_qty <= 0:
+            corr = of.get(cfg.correlation_tag)
+            if corr is None or order_qty <= 0:
                 continue
-            px = of.get(T_PRICE) or "100.00"
 
             # 1. Ack the order first (New) — immediate, not rate-limited.
             if f.send_ack:
@@ -420,37 +441,127 @@ class PerfRun:
                 if ack is not None and await self._sm.send_message(cfg.venue_session_id, ack):
                     self.stats.acks_sent += 1
 
-            # 2. Fills — gated by fill_ratio (a skipped order stays acked/open).
+            # 2. Book the fills — gated by fill_ratio (a skipped order stays
+            #    acked/open and is never booked).
             if cfg.rate.fill_ratio < 1.0 and random.random() > cfg.rate.fill_ratio:
                 continue
-            for et, ost, last_qty, cum_qty, leaves_qty in self._fill_schedule(order_qty):
-                await self._fill_bucket.acquire()
-                er = self._exec(of, exec_type=et, ord_status=ost, last_qty=last_qty,
-                                cum_qty=cum_qty, leaves_qty=leaves_qty, px=px)
-                if er is not None and await self._sm.send_message(cfg.venue_session_id, er):
-                    self.stats.fills_sent += 1
+            schedule = self._fill_schedule(order_qty)
+            if not schedule:
+                continue
+            self._venue_orders[corr] = {
+                "of": of, "schedule": schedule, "idx": 0, "cum": 0,
+                "px": of.get(T_PRICE) or "100.00", "order_qty": order_qty,
+                "pauses_done": 0, "waiting": False, "wait_deadline": 0.0,
+            }
+            self._fill_runnable.append(corr)
+            self._fill_wakeup.set()
+
+    async def _fill_sender(self) -> None:
+        """Drains all booked orders' fills round-robin through the shared fill
+        bucket — one order holding for its amend never blocks the others.
+
+        Amend-aware hold: after fill k of a still-working order, when amends
+        are enabled and k is a predicted trigger point (k % every_n_fills == 0,
+        pause cap not reached), the order waits up to amend_wait_ms for its
+        35=G; _handle_amend resumes it early, the grace check resumes it late.
+        No holds when amends are off or once injection has stopped (drain)."""
+        cfg = self.config
+        a = cfg.amend
+        book = self._venue_orders
+        runnable = self._fill_runnable
+        next_wait_check = 0.0
+        while True:
+            now = time.monotonic()
+            if now >= next_wait_check:
+                self._release_expired_waits(now)
+                next_wait_check = now + 0.05
+            if not runnable:
+                self._fill_wakeup.clear()
+                if not runnable:
+                    try:
+                        await asyncio.wait_for(self._fill_wakeup.wait(), timeout=0.05)
+                    except asyncio.TimeoutError:
+                        pass
+                    continue
+            corr = runnable.popleft()
+            entry = book.get(corr)
+            if entry is None or entry["waiting"]:
+                continue   # completed meanwhile, or re-queued while holding
+            await self._fill_bucket.acquire()
+            et, ost, last_qty, cum_qty, leaves_qty = entry["schedule"][entry["idx"]]
+            er = self._exec(entry["of"], exec_type=et, ord_status=ost, last_qty=last_qty,
+                            cum_qty=cum_qty, leaves_qty=leaves_qty, px=entry["px"])
+            if er is not None and await self._sm.send_message(cfg.venue_session_id, er):
+                self.stats.fills_sent += 1
+            entry["idx"] += 1
+            entry["cum"] = cum_qty
+            if entry["idx"] >= len(entry["schedule"]):
+                book.pop(corr, None)
+            elif (a.enabled and self._injecting
+                    and entry["idx"] % a.every_n_fills == 0
+                    and (a.max_per_order == 0 or entry["pauses_done"] < a.max_per_order)):
+                entry["waiting"] = True
+                entry["wait_deadline"] = time.monotonic() + a.amend_wait_ms / 1000.0
+            else:
+                runnable.append(corr)
+
+    def _release_expired_waits(self, now: float) -> None:
+        """Resume held orders whose 35=G never came (amend_ratio < 1, client
+        cap reached, or the amend was rejected upstream)."""
+        for corr, entry in self._venue_orders.items():
+            if entry["waiting"] and now >= entry["wait_deadline"]:
+                entry["waiting"] = False
+                entry["pauses_done"] += 1   # this trigger point is consumed
+                self.stats.amend_waits_expired += 1
+                self._fill_runnable.append(corr)
 
     async def _handle_amend(self, of: dict) -> None:
         """Venue side of 35=G: reply Replaced (150=5/39=5), optionally preceded
-        by Pending Replace (150=E/39=E). Stateless — built from the request's
-        own fields with no order book, so CumQty reads 0 even when the order is
-        partially filled (the OMS under test reads 150/39 + the ClOrdID chain).
-        Like acks, replies are immediate, not fill-rate-limited."""
+        by Pending Replace (150=E/39=E). With the order in the book the reply
+        carries the REAL CumQty/LeavesQty, the order's price is updated so the
+        remaining fills go out at the amended price, and a held order resumes
+        immediately. Unknown orders (already completed, a late G, or venue
+        mode against an external client) fall back to the stateless reply
+        (CumQty=0). Like acks, replies are immediate, not fill-rate-limited."""
         cfg = self.config
         self.stats.amends_received += 1
-        if of.get(cfg.correlation_tag) is None:
+        corr = of.get(cfg.correlation_tag)
+        if corr is None:
+            # Can't reply — there is nothing to echo for the client to correlate.
+            # Classic cause: the OMS rebuilds the 35=G downstream and drops the
+            # custom tag it passes through fine on NewOrderSingle.
+            self.stats.amends_uncorrelated += 1
+            if self.stats.amends_uncorrelated == 1:
+                self.warnings.append(
+                    f"venue received 35=G without correlation tag "
+                    f"{cfg.correlation_tag} — no replace ack sent; check the "
+                    f"counterparty passes the tag through on amends")
             return
         order_qty = _to_int(of.get(T_ORDERQTY), 0)
-        px = of.get(T_PRICE) or "0"
         orig = of.get(T_ORIGCLORDID)
+        entry = self._venue_orders.get(corr)
+        if entry is not None:
+            new_px = of.get(T_PRICE)
+            if new_px:
+                entry["px"] = new_px
+            cum = entry["cum"]
+            leaves = max(0, entry["order_qty"] - cum)
+            px = entry["px"]
+            entry["pauses_done"] += 1
+            if entry["waiting"]:
+                entry["waiting"] = False
+                self._fill_runnable.append(corr)
+                self._fill_wakeup.set()
+        else:
+            cum, leaves, px = 0, order_qty, of.get(T_PRICE) or "0"
         if cfg.amend.pending_replace:
             er = self._build_exec(of, exec_type="E", ord_status="E", last_qty=0,
-                                  cum_qty=0, leaves_qty=order_qty, px=px,
+                                  cum_qty=cum, leaves_qty=leaves, px=px,
                                   orig_clordid=orig)
             if er is not None:
                 await self._sm.send_message(cfg.venue_session_id, er)
         er = self._build_exec(of, exec_type="5", ord_status="5", last_qty=0,
-                              cum_qty=0, leaves_qty=order_qty, px=px,
+                              cum_qty=cum, leaves_qty=leaves, px=px,
                               orig_clordid=orig)
         if er is not None:
             await self._sm.send_message(cfg.venue_session_id, er)
@@ -540,10 +651,33 @@ class PerfRun:
         sub = self._client_sub
         while True:
             inb = await sub.get()
+            if inb.msg_type == "9":
+                self._on_cancel_reject(inb.fields)
+                continue
             if inb.msg_type != "8":
                 continue
             self._on_exec_report(inb.fields, inb.recv_perf_ns or time.perf_counter_ns())
 
+    def _on_cancel_reject(self, f: dict) -> None:
+        """OrderCancelReject (35=9): the counterparty refused the amend. Count
+        it (these would otherwise be invisible — only 35=8 is correlated) and
+        release the in-flight amend so the order finalizes normally instead of
+        being reaped as amends_lost."""
+        self.stats.amend_rejects += 1
+        if self.stats.amend_rejects == 1:
+            self.warnings.append(
+                "counterparty rejected an amend (35=9 OrderCancelReject) — "
+                "it may not match OrigClOrdID(41) against its own order chain")
+        corr = f.get(self.config.correlation_tag)
+        entry = self._pending.get(corr) if corr is not None else None
+        if entry is None or not entry.get("amend_inflight"):
+            return
+        entry["amend_inflight"] = False
+        entry["pending_clordid"] = None        # chain tip stays at the last accepted
+        if entry.get("completed"):
+            self._pending.pop(corr, None)
+            self._finalize(entry, "filled")
+
     def _on_exec_report(self, f: dict, now_ns: int) -> None:
         cfg = self.config
         corr = f.get(cfg.correlation_tag)
@@ -722,6 +856,7 @@ class PerfRun:
                 amends_sent=self.stats.amends_sent,
                 replaces_received=self.stats.replaces_received,
                 amends_lost=self.stats.amends_lost,
+                amend_rejects=self.stats.amend_rejects,
                 aps_live=round(aps, 1),
                 scenarios_dispatched=self.stats.scenarios_dispatched,
                 scenarios_per_sec=round(sps, 1),
@@ -735,7 +870,9 @@ class PerfRun:
                 fills_sent=self.stats.fills_sent, fps_live=round(fps, 1),
                 fill_ratio=round(self.stats.fill_ratio(), 3),
                 unfilled=max(0, self.stats.orders_sent - self.stats.completions),
-                amends_received=self.stats.amends_received)
+                amends_received=self.stats.amends_received,
+                amends_uncorrelated=self.stats.amends_uncorrelated,
+                amend_waits_expired=self.stats.amend_waits_expired)
         snap = LiveSnapshot(
             run_id=self.run_id, status=self.status, mode=self.config.mode,
             elapsed_s=round(self._elapsed_s(), 1), duration_s=self.config.test.duration,

{fixtureqa-0.4.9 → fixtureqa-0.4.11}/fixture/core/perf_models.py

@@ -66,13 +66,22 @@ class AmendConfig(BaseModel):
     the price by ± `price_offset` (alternating). `pending_replace` makes the
     in-process venue reply 150=E (Pending Replace) before 150=5 (Replaced),
     matching common real-venue behaviour. Amends never fire on the final fill
-    — use partial fills or fills_per_order > 1 to leave the order working."""
+    — use partial fills or fills_per_order > 1 to leave the order working.
+
+    When enabled, the in-process venue is amend-aware: it holds an order's
+    remaining fills at each predicted trigger point (every nth fill) for up
+    to `amend_wait_ms`, so the 35=G is answered while the order is still
+    working — with the real CumQty — and post-replace fills go out at the
+    amended price. Orders whose amend never arrives (amend_ratio < 1, cap
+    reached, or rejected upstream) resume after the grace. Disabled → no
+    holds, fills flow exactly as before."""
     enabled: bool = False
     every_n_fills: int = Field(default=1, ge=1)
     amend_ratio: float = Field(default=1.0, ge=0.0, le=1.0)
     max_per_order: int = Field(default=1, ge=0)
     price_offset: float = Field(default=0.01, gt=0)
     pending_replace: bool = False
+    amend_wait_ms: int = Field(default=2000, gt=0)
 
 
 class OrderPayloadConfig(BaseModel):
@@ -199,6 +208,7 @@ class ClientLeg(BaseModel):
     amends_sent: int = 0
     replaces_received: int = 0
     amends_lost: int = 0
+    amend_rejects: int = 0
     aps_live: float = 0.0
     scenarios_dispatched: int = 0
     scenarios_per_sec: float = 0.0
@@ -215,6 +225,8 @@ class VenueLeg(BaseModel):
     unfilled: int = 0
     venue_proc_us_p50: float = 0.0
     amends_received: int = 0
+    amends_uncorrelated: int = 0
+    amend_waits_expired: int = 0
 
 
 class SnapshotErrors(BaseModel):

{fixtureqa-0.4.9 → fixtureqa-0.4.11}/fixture/core/perf_stats.py

@@ -122,7 +122,10 @@ class PerfStats:
         self.amends_sent = 0         # client sent a 35=G
         self.replaces_received = 0   # client got a Replaced ack (150=5)
         self.amends_lost = 0         # amend in flight when its order was reaped
+        self.amend_rejects = 0       # client got an OrderCancelReject (35=9)
         self.amends_received = 0     # venue saw an inbound 35=G
+        self.amends_uncorrelated = 0  # inbound 35=G lacked the correlation tag (no reply)
+        self.amend_waits_expired = 0  # venue fill-hold released by grace, not by a 35=G
         self.lost_timeout = 0
         self.dropped = 0
         self.rejected = 0
@@ -179,7 +182,10 @@ class PerfStats:
             "amends_sent": self.amends_sent,
             "replaces_received": self.replaces_received,
             "amends_lost": self.amends_lost,
+            "amend_rejects": self.amend_rejects,
             "amends_received": self.amends_received,
+            "amends_uncorrelated": self.amends_uncorrelated,
+            "amend_waits_expired": self.amend_waits_expired,
             "lost_timeout": self.lost_timeout,
             "dropped": self.dropped,
             "rejected": self.rejected,