cfquant 0.2.51__tar.gz → 0.2.53__tar.gz

{cfquant-0.2.51 → cfquant-0.2.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cfquant
-Version: 0.2.51
+Version: 0.2.53
 Summary: QMT local bridge and xtquant-compatible Python client.
 License: MIT
 Requires-Python: >=3.8

cfquant-0.2.53/cfquant/callback_dispatcher.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+"""Serial user callbacks, independent of the RPC receive thread."""
+from collections import deque
+import logging
+import os
+import tempfile
+import threading
+import time
+
+from .protocol import decode_value, dumps_message, loads_message
+
+
+class CallbackDispatcher(object):
+    def __init__(self, dispatch, on_error=None):
+        self._dispatch = dispatch
+        self._on_error = on_error
+        self._condition = threading.Condition()
+        self._queue = deque()
+        self._generation = 0
+        self._active = False
+        self._thread = None
+        self._context = threading.local()
+        self._memory_limit = max(0, int(os.environ.get("CFQUANT_CALLBACK_MEMORY_BYTES", 64 * 1024 * 1024)))
+        self._memory_bytes = 0
+        self._spool = None
+        self._spool_size = 0
+        self._spool_live = 0
+        self._last_wait_ms = self._last_callback_ms = 0.0
+        self._last_warning = 0.0
+
+    def start(self):
+        with self._condition:
+            self._generation += 1
+            self._active = True
+            self._clear_queue()
+            return self._generation
+
+    def stop(self):
+        with self._condition:
+            self._active = False
+            self._generation += 1
+            self._clear_queue()
+            self._condition.notify_all()
+        # Never join: close/stop may be called by a callback itself.
+
+    def submit(self, generation, message, raw=None):
+        with self._condition:
+            if not self._active or generation != self._generation:
+                return
+            # Do not block the receive thread on a slow user callback. Trade
+            # events cannot be silently dropped or coalesced.
+            raw = raw if raw is not None else dumps_message(message)
+            size = len(raw) * 8 + 256  # Include the decoded dict/list object overhead.
+            self._queue.append([generation, message, time.monotonic(), raw, None, size])
+            self._memory_bytes += size
+            self._spill_queue()
+            if self._thread is None:
+                self._thread = threading.Thread(target=self._run, name="cfquant-event-callback")
+                self._thread.daemon = True
+                self._thread.start()
+            self._condition.notify()
+
+    def _clear_queue(self):
+        self._queue.clear()
+        self._memory_bytes = self._spool_size = self._spool_live = 0
+        if self._spool is not None:
+            self._spool.close()
+            self._spool = None
+
+    def _spill_queue(self):
+        # Allow one full batch in memory: a single large snapshot is not a
+        # backlog and must not make every healthy callback perform disk IO.
+        if len(self._queue) <= 1:
+            return
+        for item in self._queue:
+            if self._memory_bytes <= self._memory_limit:
+                break
+            if item[4] is not None:
+                continue
+            if self._spool is None:
+                self._spool = tempfile.TemporaryFile(prefix="cfquant-callbacks-")
+            raw = item[3].encode("utf-8") if isinstance(item[3], str) else item[3]
+            self._spool.seek(self._spool_size)
+            if self._spool.write(raw) != len(raw):
+                raise OSError("callback backlog could not be preserved")
+            item[4] = (self._spool_size, len(raw))
+            self._spool_size += len(raw)
+            self._spool_live += len(raw)
+            self._memory_bytes -= item[5]
+            item[1] = item[3] = None
+        if self._queue and len(self._queue) >= 1000 and time.monotonic() - self._last_warning >= 5:
+            self._last_warning = time.monotonic()
+            logging.getLogger(__name__).warning("callback backlog count=%s oldest_ms=%.1f spooled_bytes=%s",
+                                               len(self._queue), (time.monotonic() - self._queue[0][2]) * 1000, self._spool_live)
+
+    def _take_message(self, item):
+        if item[4] is None:
+            self._memory_bytes -= item[5]
+            return item[1], None
+        offset, size = item[4]
+        self._spool.seek(offset)
+        raw = self._spool.read(size)
+        if len(raw) != size:
+            raise OSError("callback backlog read was incomplete")
+        self._spool_live -= size
+        if self._spool_live == 0:
+            self._spool.close()
+            self._spool = None
+            self._spool_size = 0
+        elif self._spool_size > max(self._memory_limit, 1024 * 1024) + self._spool_live * 2:
+            replacement = tempfile.TemporaryFile(prefix="cfquant-callbacks-")
+            offsets = []
+            try:
+                for queued in self._queue:
+                    if queued[4] is None:
+                        continue
+                    previous, length = queued[4]
+                    self._spool.seek(previous)
+                    value = self._spool.read(length)
+                    next_offset = replacement.tell()
+                    if len(value) != length or replacement.write(value) != length:
+                        raise OSError("callback backlog compaction was incomplete")
+                    offsets.append((queued, (next_offset, length)))
+            except BaseException:
+                replacement.close()
+                raise
+            self._spool.close()
+            self._spool = replacement
+            self._spool_size = replacement.tell()
+            for queued, location in offsets:
+                queued[4] = location
+        return None, raw
+
+    def status(self):
+        with self._condition:
+            return {"queued_callbacks": len(self._queue), "estimated_memory_bytes": self._memory_bytes,
+                    "memory_limit_bytes": self._memory_limit, "spooled_bytes": self._spool_live,
+                    "oldest_age_ms": round((time.monotonic() - self._queue[0][2]) * 1000, 3) if self._queue else 0,
+                    "last_queue_ms": round(self._last_wait_ms, 3), "last_callback_ms": round(self._last_callback_ms, 3)}
+
+    def is_current_callback(self):
+        generation = getattr(self._context, "generation", None)
+        with self._condition:
+            return generation is None or (self._active and generation == self._generation)
+
+    def is_generation(self, generation):
+        with self._condition:
+            return generation == self._generation
+
+    def check_request(self):
+        if not self.is_current_callback():
+            raise RuntimeError("callback belongs to a closed cfquant connection")
+
+    def _run(self):
+        while True:
+            try:
+                with self._condition:
+                    while self._active and not self._queue:
+                        self._condition.wait()
+                    if not self._active:
+                        self._thread = None
+                        return
+                    item = self._queue.popleft()
+                    generation = item[0]
+                    message, raw = self._take_message(item)
+                if message is None:
+                    message = loads_message(raw)
+                    if message is None:
+                        raise ValueError("invalid preserved callback message")
+            except Exception as error:
+                logging.getLogger(__name__).exception("callback backlog failed; connection must be restarted")
+                with self._condition:
+                    if generation != self._generation:
+                        continue  # An old connection must not close its replacement.
+                    self._active = False
+                    self._clear_queue()
+                    self._thread = None
+                if self._on_error is not None:
+                    self._on_error(error, generation)
+                return
+            started = time.monotonic()
+            self._last_wait_ms = (started - item[2]) * 1000
+            self._context.generation = generation
+            try:
+                if self.is_current_callback():
+                    self._dispatch(message)
+            except Exception:
+                logging.getLogger(__name__).exception("callback dispatch failed event=%s", message.get("event"))
+            finally:
+                self._last_callback_ms = (time.monotonic() - started) * 1000
+                del self._context.generation
+
+
+def dispatch_callbacks(callbacks, message, is_current):
+    event = message.get("event")
+    raw_data = message.get("data")
+    meta = message.get("meta")
+    plain_quote = (isinstance(raw_data, dict) and isinstance(meta, dict)
+                   and meta.get("quote_json_plain") is True)
+    # The direct snapshot encoder emits only JSON scalars/containers, no tagged
+    # SDK values. Do not walk all symbols a second time after JSON parsing.
+    data = raw_data if plain_quote else decode_value(raw_data)
+    full_message = dict(message, data=data)
+    deliveries = [(list(callbacks.get("__event__", [])), full_message),
+                  (list(callbacks.get(event, [])), data)]
+    if event and event.startswith("quote:"):
+        quote_message = dict(full_message)
+        if quote_message.get("subscription_id") is not None and quote_message.get("subscribe_id") is None:
+            quote_message["subscribe_id"] = quote_message["subscription_id"]
+        deliveries.append((list(callbacks.get("quote", [])), quote_message))
+    for handlers, payload in deliveries:
+        for callback in handlers:
+            if not is_current():
+                return
+            try:
+                callback(payload)
+            except Exception:
+                logging.getLogger(__name__).exception("user callback failed event=%s", event)

{cfquant-0.2.51 → cfquant-0.2.53}/cfquant/client.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-import json
+import json
+import atexit
 import queue
 import socket
 import threading
@@ -7,17 +8,31 @@ import time
 
 from .config import get_config
 from .callback_dispatcher import CallbackDispatcher, dispatch_callbacks
-from .protocol import decode_value, dumps_message, loads_message, new_id, pack_request
+from .protocol import decode_value, dumps_message, loads_message, new_id, pack_request
+from .rpc_timing import RpcTiming
 
 
 class CfquantError(RuntimeError):
     pass
 
 
-class CfquantTimeout(TimeoutError):
-    pass
-
-
+class CfquantTimeout(TimeoutError):
+    pass
+
+
+def _wait_response(q, timeout):
+    """Keep long Windows waits interruptible without delaying incoming data."""
+    if timeout < 0 or threading.current_thread() is not threading.main_thread():
+        return q.get(timeout=timeout)
+    deadline = time.monotonic() + timeout
+    while True:
+        try:
+            return q.get(timeout=min(.2, max(0, deadline - time.monotonic())))
+        except queue.Empty:
+            if time.monotonic() >= deadline:
+                raise
+
+
 PIPE_TRANSPORTS = ("pipe", "ctypes", "named_pipe", "named-pipe")
 WEB_LTTX_TRANSPORTS = ("web", "web_lttx", "lttx_web", "web-lttx", "lttx-web")
 
@@ -113,9 +128,10 @@ class LTtxRpcClient(object):
         self._callbacks = {}
         self._lock = threading.RLock()
         self._pending_lock = threading.RLock()
-        self._started = False
+        self._started = False
+        self._closed = False
         self._recv_thread = None
-        self._event_dispatcher = CallbackDispatcher(self._dispatch_event)
+        self._event_dispatcher = CallbackDispatcher(self._dispatch_event, self._callback_buffer_failed)
 
     def start(self):
         with self._lock:
@@ -123,19 +139,37 @@ class LTtxRpcClient(object):
             if self._started:
                 return
             txl = self._load_txl()
-            self._tx = txl(self.host, self.port, self.token)
-            self._tx.start_tx()
-            self._tx.start_txg(self.client_id)
-            self._started = True
+            self._tx = txl(self.host, self.port, self.token)
+            try:
+                result = self._tx.start_tx()
+                if isinstance(result, dict) and result.get("code", 0) != 0:
+                    raise CfquantError(result.get("msg") or "LTtx connection failed")
+                self._tx.start_txg(self.client_id)
+                if not getattr(self._tx, "run_on", True):
+                    raise CfquantError("cfquant client closed during connection startup")
+            except BaseException:
+                self.close()
+                raise
+            self._started = True
+            self._closed = False
             generation = self._event_dispatcher.start()
             self._recv_thread = threading.Thread(target=self._recv_loop, args=(self._tx, generation))
             self._recv_thread.daemon = True
             self._recv_thread.start()
 
-    def close(self):
-        with self._lock:
-            self._event_dispatcher.stop()
-            self._started = False
+    def close(self):
+        # start() owns _lock while connecting/retrying. Cancel the underlying
+        # transport first so a close from another thread can acquire that lock.
+        starting_tx = self._tx if not self._started else None
+        if starting_tx is not None:
+            try:
+                starting_tx.close()
+            except Exception:
+                pass
+        with self._lock:
+            self._event_dispatcher.stop()
+            self._started = False
+            self._closed = True
             tx = self._tx
             self._tx = None
             if tx is not None:
@@ -143,10 +177,11 @@ class LTtxRpcClient(object):
                     tx.Q.put(None)
                 except Exception:
                     pass
-                try:
-                    tx.close()
-                except Exception:
-                    pass
+                if tx is not starting_tx:
+                    try:
+                        tx.close()
+                    except Exception:
+                        pass
             with self._pending_lock:
                 for q in list(self._pending.values()):
                     try:
@@ -155,7 +190,7 @@ class LTtxRpcClient(object):
                         pass
                 self._pending.clear()
 
-    def request(self, action, params=None, timeout=None, request_channel=None):
+    def request(self, action, params=None, timeout=None, request_channel=None, response_meta=None):
         effective_timeout = float(timeout or self.timeout)
         request_id = new_id("req")
         raw = pack_request(
@@ -167,6 +202,7 @@ class LTtxRpcClient(object):
             timeout=effective_timeout,
         )
         q = queue.Queue(maxsize=1)
+        timing = RpcTiming(response_meta, request_id)
         try:
             with self._lock:
                 self.start()
@@ -174,14 +210,19 @@ class LTtxRpcClient(object):
                     self._pending[request_id] = q
                 self._push("request", raw, request_channel or self.request_channel)
             try:
-                msg = q.get(timeout=effective_timeout)
+                msg = _wait_response(q, effective_timeout)
             except queue.Empty:
                 raise CfquantTimeout("cfquant request timeout: %s" % action)
-            if not msg.get("ok"):
+            timing.received(msg)
+            if not msg.get("ok"):
                 err = msg.get("error") or {}
                 raise CfquantError(err.get("message") or str(err))
-            return decode_value(msg.get("result"))
-        finally:
+            return timing.decode(msg.get("result"), decode_value)
+        except (KeyboardInterrupt, SystemExit):
+            self.close()
+            raise
+        finally:
+            timing.finish()
             with self._pending_lock:
                 self._pending.pop(request_id, None)
 
@@ -209,20 +250,26 @@ class LTtxRpcClient(object):
                 raw = tx.Q.get()
                 if raw is None:
                     break
-                msg = loads_message(raw)
+                parse_started = time.perf_counter()
+                msg = loads_message(raw)
                 if not msg:
                     continue
                 with self._lock:
                     if not self._started or tx is not self._tx:
                         return
                     msg_type = msg.get("type")
-                    if msg_type == "response":
+                    if msg_type == "response":
+                        msg["_client_parse_ms"] = round((time.perf_counter() - parse_started) * 1000, 3)
                         with self._pending_lock:
                             q = self._pending.pop(msg.get("id"), None)
                         if q:
                             q.put(msg)
-                    elif msg_type == "event":
-                        self._event_dispatcher.submit(generation, msg)
+                    elif msg_type == "event":
+                        try:
+                            self._event_dispatcher.submit(generation, msg, raw=raw)
+                        except Exception as error:
+                            self._callback_buffer_failed(error)
+                            return
             except Exception:
                 time.sleep(0.05)
                 if not self._started:
@@ -231,8 +278,25 @@ class LTtxRpcClient(object):
             if self._tx is expected_tx:
                 self.close()
 
-    def _dispatch_event(self, msg):
-        dispatch_callbacks(self._callbacks, msg, self._event_dispatcher.is_current_callback)
+    def _dispatch_event(self, msg):
+        dispatch_callbacks(self._callbacks, msg, self._event_dispatcher.is_current_callback)
+
+    def callback_status(self):
+        return self._event_dispatcher.status()
+
+    def _callback_buffer_failed(self, error, generation=None):
+        with self._lock:
+            if generation is not None and not self._event_dispatcher.is_generation(generation):
+                return
+            with self._pending_lock:
+                pending = list(self._pending.values())
+                self._pending.clear()
+            for waiter in pending:
+                try:
+                    waiter.put_nowait({"ok": False, "error": {"message": "cfquant callback buffering failed: %s" % error}})
+                except queue.Full:
+                    pass
+            self.close()
 
     def _push(self, key, payload, channel):
         tx = self._tx
@@ -333,7 +397,7 @@ _default_client = None
 _client_lock = threading.Lock()
 
 
-def get_client():
+def get_client():
     global _default_client
     with _client_lock:
         if _default_client is None:
@@ -343,9 +407,21 @@ def get_client():
                 timeout=cfg.get("timeout"),
                 client_id=cfg.get("client_id"),
             )
-        return _default_client
-
-
+        return _default_client
+
+
+def close_client():
+    """Close the default market-data client without creating a connection."""
+    global _default_client
+    with _client_lock:
+        client, _default_client = _default_client, None
+    if client is not None:
+        client.close()
+
+
+atexit.register(close_client)
+
+
 def configure(**kwargs):
     from .config import configure as configure_config
 

cfquant-0.2.53/cfquant/diagnostics.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""Opt-in whole-quote latency diagnostics; ordinary xtdata payloads are unchanged."""
+import datetime
+import math
+import time
+
+from . import xtdata
+
+
+SCHEMA = "cfquant.quote_latency.v1"
+ACTION = "xtdata.subscribe_whole_quote_latency"
+_SHANGHAI = datetime.timezone(datetime.timedelta(hours=8))
+
+
+def quote_time_ms(value):
+    """Accept epoch seconds/ms/us/ns and QMT calendar strings (China time).
+
+    Time-only and date-only fields are deliberately not assigned today's date.
+    """
+    if value is None or isinstance(value, bool):
+        return None
+    text = str(value).strip()
+    if text.isdigit() and len(text) in (14, 17) and text[:2] in ("19", "20"):
+        try:
+            stamp = datetime.datetime.strptime(text[:14], "%Y%m%d%H%M%S").replace(tzinfo=_SHANGHAI)
+            return stamp.timestamp() * 1000.0 + (int(text[14:]) if len(text) == 17 else 0)
+        except ValueError:
+            return None
+    try:
+        number = float(value)
+    except (ValueError, TypeError, OverflowError):
+        for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y%m%d %H:%M:%S.%f", "%Y%m%d %H:%M:%S"):
+            try:
+                return datetime.datetime.strptime(text, fmt).replace(tzinfo=_SHANGHAI).timestamp() * 1000.0
+            except ValueError:
+                pass
+        return None
+    if not math.isfinite(number) or number < 1000000000:
+        return None
+    if number >= 1e17:
+        number /= 1e6
+    elif number >= 1e14:
+        number /= 1e3
+    elif number < 1e11:
+        number *= 1000.0
+    # Reject short dates and implausible timestamps rather than report bogus latency.
+    return number if 946684800000 <= number < 4102444800000 else None
+
+
+def latency_stats(values):
+    values = sorted(value for value in values if value is not None and math.isfinite(value))
+    if not values:
+        return {"count": 0, "min_ms": None, "max_ms": None, "avg_ms": None, "p95_ms": None}
+    return {"count": len(values), "min_ms": values[0], "max_ms": values[-1],
+            "avg_ms": round(sum(values) / len(values), 3),
+            "p95_ms": values[max(0, (len(values) * 95 + 99) // 100 - 1)]}
+
+
+def latency_report(payload, received_at_ms=None, sample_limit=None):
+    """Measure at callback entry, before reporting/rendering. Never clamp negative deltas."""
+    received_at_ms = time.time() * 1000.0 if received_at_ms is None else received_at_ms
+    analysis_started = time.perf_counter()
+    if not isinstance(payload, dict) or payload.get("schema") != SCHEMA:
+        raise ValueError("not a cfquant whole-quote latency event; update the QMT entry script")
+    quotes = payload.get("quotes") or {}
+    pushed = quote_time_ms(payload.get("qmt_push_at_ms"))
+    entered = quote_time_ms(payload.get("qmt_callback_at_ms"))
+
+    def delta(end, start):
+        return round(end - start, 3) if end is not None and start is not None else None
+
+    sample_limit = None if sample_limit is None else max(1, int(sample_limit))
+    samples = []
+    for code, quote in quotes.items():
+        if sample_limit is not None and len(samples) >= sample_limit:
+            break
+        if not isinstance(quote, dict):
+            continue
+        stamp, field, raw = None, None, None
+        for name in ("time", "stime", "timetag", "datetime", "updateTime", "update_time"):
+            candidate = quote.get(name)
+            if raw is None and candidate is not None:
+                field, raw = name, candidate
+            stamp = quote_time_ms(candidate)
+            if stamp is not None:
+                field, raw = name, candidate
+                break
+        samples.append({
+            "code": str(code), "quote_time_field": field, "quote_time_raw": raw,
+            "quote_at_ms": stamp, "quote_latency_ms": delta(received_at_ms, stamp),
+            # Use the timestamp captured at QMT callback entry, before locking
+            # or conversion; receiver/formatting time must not affect this value.
+            "qmt_quote_latency_ms": delta(entered, stamp),
+            "last_price": quote.get("lastPrice", quote.get("last_price")),
+            "volume": quote.get("volume"), "amount": quote.get("amount"),
+        })
+    summary = {
+        "quote_latency": latency_stats(row["quote_latency_ms"] for row in samples),
+        "qmt_quote_latency": latency_stats(row["qmt_quote_latency_ms"] for row in samples),
+    }
+    return {
+        "schema": SCHEMA, "received_at_ms": received_at_ms,
+        "qmt_callback_at_ms": entered, "qmt_push_at_ms": pushed,
+        "internal_latency_ms": delta(received_at_ms, pushed),
+        "qmt_prepare_ms": delta(pushed, entered),
+        "qmt_encode_ms": payload.get("qmt_encode_ms"), "qmt_encoding": payload.get("qmt_encoding"),
+        "code_count": len(quotes), "samples": samples, "summary": summary,
+        "analysis_ms": round((time.perf_counter() - analysis_started) * 1000, 3),
+    }
+
+
+def subscribe_whole_quote(code_list, callback, sample_limit=None):
+    """Independent diagnostic subscription. Callback receives a measured report.
+
+    Use xtdata.unsubscribe_quote(id) to stop. Internal latency includes encoding,
+    transport, decoding and callback scheduling, ending at this Python callback.
+    """
+    if not callable(callback):
+        raise TypeError("callback must be callable")
+    sample_limit = None if sample_limit is None else max(1, int(sample_limit))
+
+    def on_quote(payload):
+        received_at_ms = time.time() * 1000.0
+        callback(latency_report(payload, received_at_ms, sample_limit))
+
+    return xtdata._subscribe_quote_request(ACTION, {"code_list": code_list}, on_quote)