mftik 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. mftik/__init__.py +10 -0
  2. mftik/broker/__init__.py +22 -0
  3. mftik/broker/client.py +554 -0
  4. mftik/broker/config.py +37 -0
  5. mftik/broker/errors.py +21 -0
  6. mftik/broker/request.py +46 -0
  7. mftik/broker/stream.py +87 -0
  8. mftik/cli/__init__.py +16 -0
  9. mftik/cli/app.py +340 -0
  10. mftik/cli/check.py +92 -0
  11. mftik/cli/client.py +277 -0
  12. mftik/cli/config.py +190 -0
  13. mftik/cli/connect.py +192 -0
  14. mftik/cli/exits.py +16 -0
  15. mftik/cli/init.py +208 -0
  16. mftik/cli/node.py +108 -0
  17. mftik/cli/output.py +41 -0
  18. mftik/cli/profiles.py +47 -0
  19. mftik/cli/push.py +51 -0
  20. mftik/cli/run.py +101 -0
  21. mftik/cli/sessions.py +75 -0
  22. mftik/cli/templates/Caddyfile +24 -0
  23. mftik/cli/templates/docker-compose.yml +206 -0
  24. mftik/cli/templates/env +53 -0
  25. mftik/cli/tree.py +71 -0
  26. mftik/exchange/__init__.py +284 -0
  27. mftik/exchange/base.py +68 -0
  28. mftik/exchange/binance/__init__.py +1 -0
  29. mftik/exchange/binance/feed.py +181 -0
  30. mftik/exchange/binance/future/__init__.py +150 -0
  31. mftik/exchange/binance/future/client.py +422 -0
  32. mftik/exchange/binance/future/feed.py +294 -0
  33. mftik/exchange/binance/future/methods.py +133 -0
  34. mftik/exchange/binance/future/models.py +1040 -0
  35. mftik/exchange/binance/future/private.py +519 -0
  36. mftik/exchange/binance/future/protocol.py +124 -0
  37. mftik/exchange/binance/future/public.py +419 -0
  38. mftik/exchange/binance/future/rest.py +289 -0
  39. mftik/exchange/binance/future/streams.py +221 -0
  40. mftik/exchange/binance/future/user.py +249 -0
  41. mftik/exchange/binance/models.py +126 -0
  42. mftik/exchange/binance/protocol.py +455 -0
  43. mftik/exchange/binance/rest.py +172 -0
  44. mftik/exchange/binance/socket.py +407 -0
  45. mftik/exchange/binance/spot/__init__.py +137 -0
  46. mftik/exchange/binance/spot/client.py +531 -0
  47. mftik/exchange/binance/spot/feed.py +157 -0
  48. mftik/exchange/binance/spot/methods.py +139 -0
  49. mftik/exchange/binance/spot/models.py +820 -0
  50. mftik/exchange/binance/spot/private.py +383 -0
  51. mftik/exchange/binance/spot/protocol.py +68 -0
  52. mftik/exchange/binance/spot/public.py +360 -0
  53. mftik/exchange/binance/spot/rest.py +148 -0
  54. mftik/exchange/binance/spot/socket.py +12 -0
  55. mftik/exchange/binance/spot/streams.py +111 -0
  56. mftik/exchange/bybit/__init__.py +152 -0
  57. mftik/exchange/bybit/account.py +269 -0
  58. mftik/exchange/bybit/channels.py +248 -0
  59. mftik/exchange/bybit/feed.py +471 -0
  60. mftik/exchange/bybit/models.py +790 -0
  61. mftik/exchange/bybit/private.py +621 -0
  62. mftik/exchange/bybit/protocol.py +609 -0
  63. mftik/exchange/bybit/public.py +392 -0
  64. mftik/exchange/bybit/rest.py +611 -0
  65. mftik/exchange/bybit/socket.py +459 -0
  66. mftik/exchange/bybit/trade.py +295 -0
  67. mftik/exchange/errors.py +21 -0
  68. mftik/exchange/gate/__init__.py +1 -0
  69. mftik/exchange/gate/spot/__init__.py +83 -0
  70. mftik/exchange/gate/spot/channels.py +103 -0
  71. mftik/exchange/gate/spot/client.py +744 -0
  72. mftik/exchange/gate/spot/models.py +455 -0
  73. mftik/exchange/gate/spot/private.py +365 -0
  74. mftik/exchange/gate/spot/protocol.py +288 -0
  75. mftik/exchange/gate/spot/public.py +317 -0
  76. mftik/exchange/gate/spot/rest.py +473 -0
  77. mftik/exchange/intervals.py +87 -0
  78. mftik/exchange/models.py +590 -0
  79. mftik/exchange/oms.py +118 -0
  80. mftik/exchange/paper/__init__.py +12 -0
  81. mftik/exchange/paper/engine.py +1015 -0
  82. mftik/exchange/paper/private.py +135 -0
  83. mftik/exchange/paper/public.py +83 -0
  84. mftik/exchange/paper/remote.py +229 -0
  85. mftik/exchange/paper/remote_public.py +124 -0
  86. mftik/exchange/stream.py +63 -0
  87. mftik/exchange/symbols.py +97 -0
  88. mftik/exchange/tickers.py +213 -0
  89. mftik/exchange/venues.py +283 -0
  90. mftik/liveness.py +94 -0
  91. mftik/protocol/__init__.py +482 -0
  92. mftik/protocol/envelope.py +69 -0
  93. mftik/protocol/messages.py +1223 -0
  94. mftik/protocol/query_codes.py +166 -0
  95. mftik/protocol/reject_codes.py +193 -0
  96. mftik/protocol/session_log.py +112 -0
  97. mftik/protocol/strategy_catalog.py +257 -0
  98. mftik/protocol/strategy_yml.py +190 -0
  99. mftik/protocol/topics.py +229 -0
  100. mftik/py.typed +1 -0
  101. mftik/registry/__init__.py +48 -0
  102. mftik/registry/digest.py +27 -0
  103. mftik/registry/errors.py +18 -0
  104. mftik/registry/files.py +82 -0
  105. mftik/registry/gate.py +309 -0
  106. mftik/registry/inspect.py +74 -0
  107. mftik/registry/load.py +143 -0
  108. mftik/registry/protocol.py +55 -0
  109. mftik/registry/qualify.py +26 -0
  110. mftik/registry/store.py +396 -0
  111. mftik/registry/sync.py +236 -0
  112. mftik/runtime.py +37 -0
  113. mftik/strategy/__init__.py +27 -0
  114. mftik/strategy/base.py +513 -0
  115. mftik/strategy/client_order_id.py +120 -0
  116. mftik/strategy/eventlog.py +491 -0
  117. mftik/strategy/ledger.py +261 -0
  118. mftik/strategy/mds.py +283 -0
  119. mftik/strategy/oms.py +381 -0
  120. mftik/strategy/session.py +47 -0
  121. mftik/strategy/symbols.py +144 -0
  122. mftik/strategy/tape.py +255 -0
  123. mftik/strategy/timer.py +226 -0
  124. mftik/symbols/__init__.py +5 -0
  125. mftik/symbols/client.py +185 -0
  126. mftik-0.0.3.dist-info/METADATA +92 -0
  127. mftik-0.0.3.dist-info/RECORD +129 -0
  128. mftik-0.0.3.dist-info/WHEEL +4 -0
  129. mftik-0.0.3.dist-info/entry_points.txt +2 -0
mftik/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """Shared MFTIK library — protocol, broker, runtime, exchange, strategy.
2
+
3
+ :mod:`mftik.strategy` is what a strategy is written against, and it is here
4
+ rather than in the STS app so it installs beside a strategy on a developer's
5
+ machine. Nothing in it needs a database or a running node.
6
+ """
7
+
8
+ from mftik.runtime import configure_logging, run_heartbeat_service
9
+
10
+ __all__ = ["configure_logging", "run_heartbeat_service"]
@@ -0,0 +1,22 @@
1
+ """Async Redis broker — pub/sub, request-reply, and bidirectional streams."""
2
+
3
+ from mftik.broker.client import Broker, BrokerClient
4
+ from mftik.broker.config import BrokerConfig
5
+ from mftik.broker.errors import (
6
+ BrokerError,
7
+ BrokerNotConnectedError,
8
+ RequestTimeoutError,
9
+ )
10
+ from mftik.broker.request import IncomingRequest
11
+ from mftik.broker.stream import BidirectionalStream
12
+
13
+ __all__ = [
14
+ "BidirectionalStream",
15
+ "Broker",
16
+ "BrokerClient",
17
+ "BrokerConfig",
18
+ "BrokerError",
19
+ "BrokerNotConnectedError",
20
+ "IncomingRequest",
21
+ "RequestTimeoutError",
22
+ ]
mftik/broker/client.py ADDED
@@ -0,0 +1,554 @@
1
+ """Async Redis broker — pub/sub and request-reply IPC."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import json
7
+ import logging
8
+ import time
9
+ from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
10
+ from typing import Any
11
+
12
+ import redis.asyncio as redis
13
+ from pydantic import BaseModel
14
+
15
+ from mftik.broker.config import BrokerConfig
16
+ from mftik.broker.errors import BrokerNotConnectedError, RequestTimeoutError
17
+ from mftik.broker.request import IncomingRequest
18
+ from mftik.broker.stream import BidirectionalStream
19
+ from mftik.protocol import (
20
+ Envelope,
21
+ Heartbeat,
22
+ HeartbeatEnvelope,
23
+ Topics,
24
+ UntypedEnvelope,
25
+ )
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+ Handler = Callable[[IncomingRequest], Awaitable[None]]
30
+
31
+
32
+ def _to_json(value: BaseModel | dict[str, Any]) -> str:
33
+ if isinstance(value, BaseModel):
34
+ return value.model_dump_json()
35
+ return json.dumps(value, default=str)
36
+
37
+
38
+ class Broker:
39
+ """Async Redis IPC client.
40
+
41
+ Three primitives:
42
+
43
+ 1. **Pub/Sub** — fan-out broadcast via Redis Pub/Sub
44
+ (``publish`` / ``subscribe``).
45
+ 2. **Request-reply** — 1:1 RPC via Redis lists
46
+ (``request`` / ``serve``).
47
+ 3. **Bidirectional stream** — duplex channel = pub + sub
48
+ (``bistream``).
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ config: BrokerConfig | None = None,
54
+ *,
55
+ redis_client: redis.Redis | None = None,
56
+ ) -> None:
57
+ self.config = config or BrokerConfig.from_env()
58
+ self._redis = redis_client
59
+ self._owns_redis = redis_client is None
60
+
61
+ # --- lifecycle ---------------------------------------------------------
62
+
63
+ @property
64
+ def redis(self) -> redis.Redis:
65
+ if self._redis is None:
66
+ raise BrokerNotConnectedError(
67
+ "Broker is not connected; call connect() first"
68
+ )
69
+ return self._redis
70
+
71
+ async def connect(self) -> None:
72
+ if self._redis is None:
73
+ self._redis = redis.from_url(
74
+ self.config.redis_url,
75
+ decode_responses=True,
76
+ # Pooled connections are handed out newest-first, so one that
77
+ # sinks to the bottom of the pool can idle past the server's
78
+ # ``timeout`` and be closed there. Nothing notices until it is
79
+ # borrowed again, and then the command fails on a socket that
80
+ # was already gone — which is how a domain gets a burst of
81
+ # ConnectionErrors on a Redis that is perfectly healthy.
82
+ # Checking a connection's health on checkout retires those
83
+ # before a caller can trip over one.
84
+ health_check_interval=self.config.health_check_interval,
85
+ socket_keepalive=True,
86
+ )
87
+ self._owns_redis = True
88
+ await self._redis.ping()
89
+ logger.info("Connected to Redis at %s", self.config.redis_url)
90
+
91
+ async def close(self) -> None:
92
+ if self._redis is not None and self._owns_redis:
93
+ await self._redis.aclose()
94
+ self._redis = None
95
+
96
+ async def __aenter__(self) -> Broker:
97
+ await self.connect()
98
+ return self
99
+
100
+ async def __aexit__(self, *args: object) -> None:
101
+ await self.close()
102
+
103
+ # --- key helpers -------------------------------------------------------
104
+
105
+ def _rpc_queue(self, subject: str) -> str:
106
+ return f"{self.config.key_prefix}:rpc:{subject}"
107
+
108
+ def _rpc_reply(self, request_id: str) -> str:
109
+ return f"{self.config.key_prefix}:rpc:reply:{request_id}"
110
+
111
+ def _log_buffer_key(self, topic: str) -> str:
112
+ return f"{self.config.key_prefix}:logbuf:{topic}"
113
+
114
+ def state_key(self, name: str) -> str:
115
+ """Redis key backing a shared state hash (e.g. ``td.ledger.7``)."""
116
+ return f"{self.config.key_prefix}:state:{name}"
117
+
118
+ # --- shared state (hashes) ---------------------------------------------
119
+ #
120
+ # Pub/Sub tells a reader that something changed; these hold what it
121
+ # changed *to*. A late subscriber, a restarted process and a strategy that
122
+ # missed a message all read the same current answer here, which is what
123
+ # makes "the writer's state and the reader's state agree" true by
124
+ # construction rather than by both sides keeping their own copy in sync.
125
+
126
+ async def state_put(
127
+ self, name: str, field: str, value: BaseModel | dict[str, Any]
128
+ ) -> None:
129
+ """Write one field of a state hash."""
130
+ await self.redis.hset( # type: ignore[misc]
131
+ self.state_key(name), field, _to_json(value)
132
+ )
133
+
134
+ async def state_put_many(
135
+ self, name: str, values: Mapping[str, BaseModel | dict[str, Any]]
136
+ ) -> None:
137
+ """Write several fields in one round trip."""
138
+ if not values:
139
+ return
140
+ await self.redis.hset( # type: ignore[misc]
141
+ self.state_key(name),
142
+ mapping={k: _to_json(v) for k, v in values.items()},
143
+ )
144
+
145
+ async def state_replace(
146
+ self, name: str, values: Mapping[str, BaseModel | dict[str, Any]]
147
+ ) -> None:
148
+ """Make the hash exactly ``values`` — the recon path.
149
+
150
+ Delete and rewrite run in one transaction so a reader never observes
151
+ the empty gap between them.
152
+ """
153
+ key = self.state_key(name)
154
+ pipe = self.redis.pipeline(transaction=True)
155
+ pipe.delete(key)
156
+ if values:
157
+ pipe.hset(key, mapping={k: _to_json(v) for k, v in values.items()})
158
+ await pipe.execute()
159
+
160
+ async def state_get(self, name: str, field: str) -> dict[str, Any] | None:
161
+ raw = await self.redis.hget(self.state_key(name), field) # type: ignore[misc]
162
+ return None if raw is None else json.loads(raw)
163
+
164
+ async def state_all(self, name: str) -> dict[str, dict[str, Any]]:
165
+ rows = await self.redis.hgetall(self.state_key(name)) # type: ignore[misc]
166
+ return {field: json.loads(raw) for field, raw in rows.items()}
167
+
168
+ async def state_drop(self, name: str, *fields: str) -> int:
169
+ if not fields:
170
+ return 0
171
+ return int(
172
+ await self.redis.hdel(self.state_key(name), *fields) # type: ignore[misc]
173
+ )
174
+
175
+ async def state_clear(self, *names: str) -> None:
176
+ """Delete whole state hashes — call when their owner goes away.
177
+
178
+ State that outlives its writer is worse than no state: a reader cannot
179
+ tell a stale answer from a current one.
180
+ """
181
+ if names:
182
+ await self.redis.delete(*(self.state_key(n) for n in names))
183
+
184
+ # --- recorded tape (streams) -------------------------------------------
185
+ #
186
+ # A feed's own history, kept so a strategy that starts later can warm up on
187
+ # what it missed. Streams rather than lists because the retention policy is
188
+ # a *duration* — a stream id is a millisecond timestamp, so "keep two
189
+ # hours" is ``XTRIM MINID`` and "read from T" is ``XRANGE``, neither of
190
+ # which a list can express: ``LTRIM`` counts entries, and the same count is
191
+ # eight hours of a quiet instrument or twenty minutes of a busy one.
192
+ #
193
+ # Two bounds, and they mean different things. ``maxlen`` on append is the
194
+ # memory fuse — approximate, so Redis trims whole nodes and the write stays
195
+ # cheap. The MINID trim is the intent. Whichever binds first is what the
196
+ # reader gets, and :meth:`tape_coverage` is how it finds out which.
197
+
198
+ def tape_key(self, feed: str) -> str:
199
+ """Redis stream holding recorded tape for ``feed``."""
200
+ return f"{self.config.key_prefix}:tape:{feed}"
201
+
202
+ def tape_coverage_key(self, feed: str) -> str:
203
+ """Redis hash describing what :meth:`tape_key` currently covers."""
204
+ return f"{self.config.key_prefix}:tape:coverage:{feed}"
205
+
206
+ async def tape_append(
207
+ self,
208
+ feed: str,
209
+ fields: Mapping[str, str],
210
+ *,
211
+ maxlen: int,
212
+ ttl_seconds: int,
213
+ ) -> None:
214
+ """Append one record, capping the stream at ``maxlen`` entries.
215
+
216
+ The id is Redis' own clock, not the venue's timestamp. Event time is a
217
+ field on the record instead, because ``XADD`` refuses an id that does
218
+ not exceed the last one and a venue tape is not strictly monotonic —
219
+ one late print out of a million would otherwise end the recording.
220
+
221
+ ``ttl_seconds`` is renewed on every append, so a feed that stops being
222
+ recorded expires on its own. Without it a tape would outlive the last
223
+ strategy that ever wanted it: the MINID trim only runs against feeds
224
+ that are still pumping, and a stream nobody writes to is never capped
225
+ by ``maxlen`` either. Every instrument ever subscribed would keep its
226
+ last two hours for as long as Redis lived.
227
+ """
228
+ pipe = self.redis.pipeline()
229
+ pipe.xadd(
230
+ self.tape_key(feed),
231
+ dict(fields),
232
+ maxlen=maxlen,
233
+ approximate=True,
234
+ )
235
+ pipe.expire(self.tape_key(feed), ttl_seconds)
236
+ pipe.expire(self.tape_coverage_key(feed), ttl_seconds)
237
+ await pipe.execute()
238
+
239
+ async def tape_tail(
240
+ self, feed: str, *, count: int
241
+ ) -> list[tuple[str, dict[str, str]]]:
242
+ """Read the newest ``count`` records, oldest → newest.
243
+
244
+ The newest rather than the oldest: warming up means catching up to now,
245
+ and a stream capped by two independent bounds holds an unknown number
246
+ of records, so "the first N" is not a window anyone asked for.
247
+ """
248
+ if count <= 0:
249
+ return []
250
+ rows = await self.redis.xrevrange(
251
+ self.tape_key(feed), max="+", min="-", count=count
252
+ )
253
+ return [(str(rid), dict(fields)) for rid, fields in reversed(rows)]
254
+
255
+ async def tape_trim_before(self, feed: str, *, min_id_ms: int) -> int:
256
+ """Drop records older than ``min_id_ms``. Returns how many went."""
257
+ return int(
258
+ await self.redis.xtrim(self.tape_key(feed), minid=min_id_ms)
259
+ )
260
+
261
+ async def tape_mark_recording(
262
+ self, feed: str, *, since_ms: int, ttl_seconds: int
263
+ ) -> None:
264
+ """Record that this feed started recording at ``since_ms``.
265
+
266
+ Called when a feed begins pumping, which is also the moment continuity
267
+ breaks: whatever is already in the stream predates a gap of unknown
268
+ length. Readers compare against this rather than assuming the records
269
+ they can see form one series.
270
+
271
+ Carries its own TTL because a feed can be subscribed and then print
272
+ nothing at all — a dead instrument, a venue outage — and the appends
273
+ that would otherwise renew it never come.
274
+ """
275
+ pipe = self.redis.pipeline()
276
+ pipe.hset(
277
+ self.tape_coverage_key(feed),
278
+ mapping={
279
+ "continuous_since_ms": str(since_ms),
280
+ "recording": "1",
281
+ "stopped_ms": "",
282
+ },
283
+ )
284
+ pipe.expire(self.tape_coverage_key(feed), ttl_seconds)
285
+ await pipe.execute()
286
+
287
+ async def tape_mark_stopped(self, feed: str, *, at_ms: int) -> None:
288
+ """Record that this feed stopped recording at ``at_ms``.
289
+
290
+ The stream is left alone. A reader that wants the last two hours before
291
+ a feed went quiet can still have them — it just has to know they end,
292
+ and that is exactly what this says.
293
+ """
294
+ await self.redis.hset( # type: ignore[misc]
295
+ self.tape_coverage_key(feed),
296
+ mapping={"recording": "0", "stopped_ms": str(at_ms)},
297
+ )
298
+
299
+ async def tape_coverage(self, feed: str) -> dict[str, str]:
300
+ """What :meth:`tape_key` covers, or ``{}`` if it was never recorded."""
301
+ return dict(await self.redis.hgetall(self.tape_coverage_key(feed))) # type: ignore[misc]
302
+
303
+ # --- Pub/Sub -----------------------------------------------------------
304
+
305
+ async def publish(self, topic: str, envelope: Envelope[Any]) -> int:
306
+ """Publish an envelope to a pub/sub topic (fan-out)."""
307
+ return int(await self.redis.publish(topic, envelope.to_json()))
308
+
309
+ async def publish_log(
310
+ self,
311
+ topic: str,
312
+ envelope: Envelope[Any],
313
+ *,
314
+ maxlen: int | None = None,
315
+ ttl_seconds: int = 86_400,
316
+ ) -> int:
317
+ """Publish a log line and append it to a Redis list for late subscribers.
318
+
319
+ Redis Pub/Sub alone drops messages when nobody is listening (e.g. UI
320
+ opens ``/ws/sts/...`` after deploy). The buffer is replayed on connect.
321
+ ``maxlen`` defaults to :attr:`BrokerConfig.log_buffer_maxlen` (100).
322
+ """
323
+ keep = (
324
+ self.config.log_buffer_maxlen if maxlen is None else max(1, maxlen)
325
+ )
326
+ raw = envelope.to_json()
327
+ key = self._log_buffer_key(topic)
328
+ pipe = self.redis.pipeline()
329
+ pipe.rpush(key, raw)
330
+ pipe.ltrim(key, -keep, -1)
331
+ pipe.expire(key, ttl_seconds)
332
+ pipe.publish(topic, raw)
333
+ results = await pipe.execute()
334
+ return int(results[-1])
335
+
336
+ async def fetch_log_buffer(self, topic: str) -> list[str]:
337
+ """Return buffered log JSON lines for ``topic`` (oldest → newest)."""
338
+ rows = await self.redis.lrange(self._log_buffer_key(topic), 0, -1)
339
+ return list(rows)
340
+
341
+ async def subscribe(
342
+ self,
343
+ topics: str | Sequence[str],
344
+ *,
345
+ stop: asyncio.Event | None = None,
346
+ ) -> AsyncIterator[UntypedEnvelope]:
347
+ """Yield envelopes from one or more pub/sub topics until ``stop``.
348
+
349
+ Uses Redis Pub/Sub. Messages published while not subscribed are lost
350
+ unless they were also written via :meth:`publish_log`.
351
+ """
352
+ channel_list = (topics,) if isinstance(topics, str) else tuple(topics)
353
+ if not channel_list:
354
+ raise ValueError("subscribe requires at least one topic")
355
+
356
+ pubsub = self.redis.pubsub()
357
+ await pubsub.subscribe(*channel_list)
358
+ try:
359
+ while stop is None or not stop.is_set():
360
+ message = await pubsub.get_message(
361
+ ignore_subscribe_messages=True, timeout=1.0
362
+ )
363
+ if message is None:
364
+ await asyncio.sleep(0.01)
365
+ continue
366
+ data = message.get("data")
367
+ if data is None:
368
+ continue
369
+ yield UntypedEnvelope.from_json(data)
370
+ finally:
371
+ await pubsub.unsubscribe(*channel_list)
372
+ await pubsub.aclose()
373
+
374
+ async def psubscribe(
375
+ self,
376
+ patterns: str | Sequence[str],
377
+ *,
378
+ stop: asyncio.Event | None = None,
379
+ ) -> AsyncIterator[tuple[str, UntypedEnvelope]]:
380
+ """Yield ``(channel, envelope)`` from pattern subscriptions until ``stop``.
381
+
382
+ Uses Redis ``PSUBSCRIBE``. Messages published while not subscribed are
383
+ lost unless they were also written via :meth:`publish_log`.
384
+ """
385
+ pattern_list = (patterns,) if isinstance(patterns, str) else tuple(patterns)
386
+ if not pattern_list:
387
+ raise ValueError("psubscribe requires at least one pattern")
388
+
389
+ pubsub = self.redis.pubsub()
390
+ await pubsub.psubscribe(*pattern_list)
391
+ try:
392
+ while stop is None or not stop.is_set():
393
+ message = await pubsub.get_message(
394
+ ignore_subscribe_messages=True, timeout=1.0
395
+ )
396
+ if message is None:
397
+ await asyncio.sleep(0.01)
398
+ continue
399
+ if message.get("type") != "pmessage":
400
+ continue
401
+ data = message.get("data")
402
+ channel = message.get("channel")
403
+ if data is None or channel is None:
404
+ continue
405
+ yield str(channel), UntypedEnvelope.from_json(data)
406
+ finally:
407
+ await pubsub.punsubscribe(*pattern_list)
408
+ await pubsub.aclose()
409
+
410
+ def bistream(
411
+ self,
412
+ *,
413
+ tx: str,
414
+ rx: str,
415
+ ) -> BidirectionalStream:
416
+ """Open a bidirectional stream (publish on ``tx``, subscribe on ``rx``)."""
417
+ return BidirectionalStream(self, tx=tx, rx=rx)
418
+
419
+ def bistream_pair(
420
+ self,
421
+ name: str,
422
+ ) -> tuple[BidirectionalStream, BidirectionalStream]:
423
+ """Open both ends of a named bistream: ``(up, down)``.
424
+
425
+ ``up`` publishes ``bistream.{name}.up`` and receives ``.down``;
426
+ ``down`` is the complement.
427
+ """
428
+ up_topic, down_topic = BidirectionalStream.topics(name)
429
+ up = self.bistream(tx=up_topic, rx=down_topic)
430
+ down = self.bistream(tx=down_topic, rx=up_topic)
431
+ return up, down
432
+
433
+ # --- Request-reply -----------------------------------------------------
434
+
435
+ async def request(
436
+ self,
437
+ subject: str,
438
+ envelope: Envelope[Any],
439
+ *,
440
+ timeout: float | None = None,
441
+ ) -> UntypedEnvelope:
442
+ """Send a request and wait for a single reply.
443
+
444
+ The envelope's ``id`` is used as the correlation id. A temporary
445
+ reply list key is written into ``reply_to`` before enqueueing.
446
+ """
447
+ wait = self.config.request_timeout if timeout is None else timeout
448
+ reply_key = self._rpc_reply(envelope.id)
449
+ outbound = (
450
+ envelope
451
+ if envelope.reply_to == reply_key
452
+ else envelope.model_copy(update={"reply_to": reply_key})
453
+ )
454
+
455
+ queue = self._rpc_queue(subject)
456
+ await self.redis.rpush(queue, outbound.to_json())
457
+
458
+ # BLPOP timeout is whole seconds; poll until the deadline for accuracy.
459
+ deadline = time.monotonic() + wait
460
+ try:
461
+ while True:
462
+ remaining = deadline - time.monotonic()
463
+ if remaining <= 0:
464
+ raise RequestTimeoutError(subject, outbound.id, wait)
465
+ result = await self.redis.blpop(reply_key, timeout=1)
466
+ if result is None:
467
+ continue
468
+ _key, data = result
469
+ return UntypedEnvelope.from_json(data)
470
+ finally:
471
+ await self.redis.delete(reply_key)
472
+
473
+ async def post(self, subject: str, envelope: Envelope[Any]) -> None:
474
+ """Enqueue on a request-reply subject without waiting for a reply.
475
+
476
+ The same queue :meth:`request` uses and the same competing consumers
477
+ take from it; what is missing is the ``reply_to``, so the handler
478
+ answers nobody and this returns as soon as Redis has the message.
479
+
480
+ For work whose *result* the sender has no use for and whose duration it
481
+ must not inherit — a backfill run is minutes of venue round trips, and
482
+ the shutdown path that asks for one is measured in seconds. A request
483
+ left in the list because nothing is serving the subject yet is not lost:
484
+ the next consumer to come up takes it, which is the recovery a pub/sub
485
+ message could not offer.
486
+ """
487
+ await self.redis.rpush(self._rpc_queue(subject), envelope.to_json())
488
+
489
+ async def serve(
490
+ self,
491
+ subject: str,
492
+ *,
493
+ stop: asyncio.Event | None = None,
494
+ ) -> AsyncIterator[IncomingRequest]:
495
+ """Yield incoming requests on a request-reply subject.
496
+
497
+ Call ``await req.reply(envelope)`` to respond. Competing consumers
498
+ on the same subject share work via Redis list ``BLPOP``.
499
+ """
500
+ queue = self._rpc_queue(subject)
501
+ while stop is None or not stop.is_set():
502
+ result = await self.redis.blpop(queue, timeout=1)
503
+ if result is None:
504
+ continue
505
+ _key, data = result
506
+ envelope = UntypedEnvelope.from_json(data)
507
+ yield IncomingRequest(self, envelope)
508
+
509
+ async def serve_handler(
510
+ self,
511
+ subject: str,
512
+ handler: Handler,
513
+ *,
514
+ stop: asyncio.Event | None = None,
515
+ ) -> None:
516
+ """Run ``handler`` for each incoming request until ``stop``."""
517
+ async for req in self.serve(subject, stop=stop):
518
+ await handler(req)
519
+
520
+ async def _send_reply(self, reply_to: str, envelope: Envelope[Any]) -> None:
521
+ await self.redis.rpush(reply_to, envelope.to_json())
522
+ await self.redis.expire(reply_to, self.config.reply_ttl_seconds)
523
+
524
+ # --- convenience -------------------------------------------------------
525
+
526
+ async def heartbeat_loop(
527
+ self,
528
+ source: str,
529
+ *,
530
+ interval: float = 5.0,
531
+ stop: asyncio.Event | None = None,
532
+ on_tick: Callable[[], None] | None = None,
533
+ ) -> None:
534
+ """Publish periodic heartbeats on the heartbeat pub/sub topic."""
535
+ while stop is None or not stop.is_set():
536
+ envelope = HeartbeatEnvelope.wrap(
537
+ Heartbeat(),
538
+ type="heartbeat",
539
+ source=source,
540
+ )
541
+ await self.publish(Topics.HEARTBEAT, envelope)
542
+ if on_tick is not None:
543
+ on_tick()
544
+ try:
545
+ if stop is not None:
546
+ await asyncio.wait_for(stop.wait(), timeout=interval)
547
+ else:
548
+ await asyncio.sleep(interval)
549
+ except TimeoutError:
550
+ continue
551
+
552
+
553
+ # Back-compat alias used during the rename.
554
+ BrokerClient = Broker
mftik/broker/config.py ADDED
@@ -0,0 +1,37 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class BrokerConfig:
9
+ """Redis broker connection and IPC defaults."""
10
+
11
+ redis_url: str = "redis://localhost:6379/0"
12
+ key_prefix: str = "mft"
13
+ request_timeout: float = 5.0
14
+ reply_ttl_seconds: int = 60
15
+ #: How many log lines ``publish_log`` keeps per topic for late WS
16
+ #: subscribers. Older lines are trimmed; live pub/sub is unaffected.
17
+ log_buffer_maxlen: int = 100
18
+ #: How old a pooled connection may be before it is pinged on checkout.
19
+ #: Must stay under the Redis server's ``timeout`` (300s in production) —
20
+ #: the point is to retire a connection the server has already closed
21
+ #: before a caller borrows it and fails on it.
22
+ health_check_interval: int = 30
23
+
24
+ @classmethod
25
+ def from_env(cls) -> BrokerConfig:
26
+ return cls(
27
+ redis_url=os.getenv("REDIS_URL", "redis://localhost:6379/0"),
28
+ key_prefix=os.getenv("BROKER_KEY_PREFIX", "mft"),
29
+ request_timeout=float(os.getenv("BROKER_REQUEST_TIMEOUT", "5")),
30
+ reply_ttl_seconds=int(os.getenv("BROKER_REPLY_TTL", "60")),
31
+ log_buffer_maxlen=max(
32
+ 1, int(os.getenv("BROKER_LOG_BUFFER_MAXLEN", "100"))
33
+ ),
34
+ health_check_interval=int(
35
+ os.getenv("BROKER_HEALTH_CHECK_INTERVAL", "30")
36
+ ),
37
+ )
mftik/broker/errors.py ADDED
@@ -0,0 +1,21 @@
1
+ """Broker IPC errors."""
2
+
3
+
4
+ class BrokerError(Exception):
5
+ """Base error for broker operations."""
6
+
7
+
8
+ class BrokerNotConnectedError(BrokerError):
9
+ """Raised when an operation is attempted before connect()."""
10
+
11
+
12
+ class RequestTimeoutError(BrokerError):
13
+ """Raised when a request-reply call exceeds its timeout."""
14
+
15
+ def __init__(self, subject: str, request_id: str, timeout: float) -> None:
16
+ self.subject = subject
17
+ self.request_id = request_id
18
+ self.timeout = timeout
19
+ super().__init__(
20
+ f"request to {subject!r} timed out after {timeout}s (id={request_id})"
21
+ )
@@ -0,0 +1,46 @@
1
+ """Request-reply request handle."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING, Any
6
+
7
+ from mftik.protocol import Envelope, UntypedEnvelope
8
+
9
+ if TYPE_CHECKING:
10
+ from mftik.broker.client import Broker
11
+
12
+
13
+ class IncomingRequest:
14
+ """A request waiting for a reply on a request-reply subject."""
15
+
16
+ __slots__ = ("envelope", "_broker", "_replied")
17
+
18
+ def __init__(self, broker: Broker, envelope: UntypedEnvelope) -> None:
19
+ self.envelope = envelope
20
+ self._broker = broker
21
+ self._replied = False
22
+
23
+ @property
24
+ def replied(self) -> bool:
25
+ return self._replied
26
+
27
+ async def reply(self, envelope: Envelope[Any]) -> None:
28
+ """Send a reply envelope to the requester's reply inbox, if there is one.
29
+
30
+ A missing ``reply_to`` is not an error. It is what
31
+ :meth:`~mftik.broker.client.Broker.post` produces — the same queue and
32
+ the same handlers as :meth:`~mftik.broker.client.Broker.request`, minus
33
+ anybody waiting — so a handler that always replies is exactly what
34
+ makes a subject postable. Raising here would have meant every such
35
+ handler needed a guard, and forgetting one would surface only on the
36
+ posted path, after the work was already done.
37
+
38
+ :attr:`replied` distinguishes the two afterwards: a handler that wants
39
+ to skip building an answer nobody will read can check ``reply_to``
40
+ itself first.
41
+ """
42
+ reply_to = self.envelope.reply_to
43
+ if not reply_to:
44
+ return
45
+ await self._broker._send_reply(reply_to, envelope)
46
+ self._replied = True