mftik 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mftik/__init__.py +10 -0
- mftik/broker/__init__.py +22 -0
- mftik/broker/client.py +554 -0
- mftik/broker/config.py +37 -0
- mftik/broker/errors.py +21 -0
- mftik/broker/request.py +46 -0
- mftik/broker/stream.py +87 -0
- mftik/cli/__init__.py +16 -0
- mftik/cli/app.py +340 -0
- mftik/cli/check.py +92 -0
- mftik/cli/client.py +277 -0
- mftik/cli/config.py +190 -0
- mftik/cli/connect.py +192 -0
- mftik/cli/exits.py +16 -0
- mftik/cli/init.py +208 -0
- mftik/cli/node.py +108 -0
- mftik/cli/output.py +41 -0
- mftik/cli/profiles.py +47 -0
- mftik/cli/push.py +51 -0
- mftik/cli/run.py +101 -0
- mftik/cli/sessions.py +75 -0
- mftik/cli/templates/Caddyfile +24 -0
- mftik/cli/templates/docker-compose.yml +206 -0
- mftik/cli/templates/env +53 -0
- mftik/cli/tree.py +71 -0
- mftik/exchange/__init__.py +284 -0
- mftik/exchange/base.py +68 -0
- mftik/exchange/binance/__init__.py +1 -0
- mftik/exchange/binance/feed.py +181 -0
- mftik/exchange/binance/future/__init__.py +150 -0
- mftik/exchange/binance/future/client.py +422 -0
- mftik/exchange/binance/future/feed.py +294 -0
- mftik/exchange/binance/future/methods.py +133 -0
- mftik/exchange/binance/future/models.py +1040 -0
- mftik/exchange/binance/future/private.py +519 -0
- mftik/exchange/binance/future/protocol.py +124 -0
- mftik/exchange/binance/future/public.py +419 -0
- mftik/exchange/binance/future/rest.py +289 -0
- mftik/exchange/binance/future/streams.py +221 -0
- mftik/exchange/binance/future/user.py +249 -0
- mftik/exchange/binance/models.py +126 -0
- mftik/exchange/binance/protocol.py +455 -0
- mftik/exchange/binance/rest.py +172 -0
- mftik/exchange/binance/socket.py +407 -0
- mftik/exchange/binance/spot/__init__.py +137 -0
- mftik/exchange/binance/spot/client.py +531 -0
- mftik/exchange/binance/spot/feed.py +157 -0
- mftik/exchange/binance/spot/methods.py +139 -0
- mftik/exchange/binance/spot/models.py +820 -0
- mftik/exchange/binance/spot/private.py +383 -0
- mftik/exchange/binance/spot/protocol.py +68 -0
- mftik/exchange/binance/spot/public.py +360 -0
- mftik/exchange/binance/spot/rest.py +148 -0
- mftik/exchange/binance/spot/socket.py +12 -0
- mftik/exchange/binance/spot/streams.py +111 -0
- mftik/exchange/bybit/__init__.py +152 -0
- mftik/exchange/bybit/account.py +269 -0
- mftik/exchange/bybit/channels.py +248 -0
- mftik/exchange/bybit/feed.py +471 -0
- mftik/exchange/bybit/models.py +790 -0
- mftik/exchange/bybit/private.py +621 -0
- mftik/exchange/bybit/protocol.py +609 -0
- mftik/exchange/bybit/public.py +392 -0
- mftik/exchange/bybit/rest.py +611 -0
- mftik/exchange/bybit/socket.py +459 -0
- mftik/exchange/bybit/trade.py +295 -0
- mftik/exchange/errors.py +21 -0
- mftik/exchange/gate/__init__.py +1 -0
- mftik/exchange/gate/spot/__init__.py +83 -0
- mftik/exchange/gate/spot/channels.py +103 -0
- mftik/exchange/gate/spot/client.py +744 -0
- mftik/exchange/gate/spot/models.py +455 -0
- mftik/exchange/gate/spot/private.py +365 -0
- mftik/exchange/gate/spot/protocol.py +288 -0
- mftik/exchange/gate/spot/public.py +317 -0
- mftik/exchange/gate/spot/rest.py +473 -0
- mftik/exchange/intervals.py +87 -0
- mftik/exchange/models.py +590 -0
- mftik/exchange/oms.py +118 -0
- mftik/exchange/paper/__init__.py +12 -0
- mftik/exchange/paper/engine.py +1015 -0
- mftik/exchange/paper/private.py +135 -0
- mftik/exchange/paper/public.py +83 -0
- mftik/exchange/paper/remote.py +229 -0
- mftik/exchange/paper/remote_public.py +124 -0
- mftik/exchange/stream.py +63 -0
- mftik/exchange/symbols.py +97 -0
- mftik/exchange/tickers.py +213 -0
- mftik/exchange/venues.py +283 -0
- mftik/liveness.py +94 -0
- mftik/protocol/__init__.py +482 -0
- mftik/protocol/envelope.py +69 -0
- mftik/protocol/messages.py +1223 -0
- mftik/protocol/query_codes.py +166 -0
- mftik/protocol/reject_codes.py +193 -0
- mftik/protocol/session_log.py +112 -0
- mftik/protocol/strategy_catalog.py +257 -0
- mftik/protocol/strategy_yml.py +190 -0
- mftik/protocol/topics.py +229 -0
- mftik/py.typed +1 -0
- mftik/registry/__init__.py +48 -0
- mftik/registry/digest.py +27 -0
- mftik/registry/errors.py +18 -0
- mftik/registry/files.py +82 -0
- mftik/registry/gate.py +309 -0
- mftik/registry/inspect.py +74 -0
- mftik/registry/load.py +143 -0
- mftik/registry/protocol.py +55 -0
- mftik/registry/qualify.py +26 -0
- mftik/registry/store.py +396 -0
- mftik/registry/sync.py +236 -0
- mftik/runtime.py +37 -0
- mftik/strategy/__init__.py +27 -0
- mftik/strategy/base.py +513 -0
- mftik/strategy/client_order_id.py +120 -0
- mftik/strategy/eventlog.py +491 -0
- mftik/strategy/ledger.py +261 -0
- mftik/strategy/mds.py +283 -0
- mftik/strategy/oms.py +381 -0
- mftik/strategy/session.py +47 -0
- mftik/strategy/symbols.py +144 -0
- mftik/strategy/tape.py +255 -0
- mftik/strategy/timer.py +226 -0
- mftik/symbols/__init__.py +5 -0
- mftik/symbols/client.py +185 -0
- mftik-0.0.3.dist-info/METADATA +92 -0
- mftik-0.0.3.dist-info/RECORD +129 -0
- mftik-0.0.3.dist-info/WHEEL +4 -0
- mftik-0.0.3.dist-info/entry_points.txt +2 -0
mftik/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Shared MFTIK library — protocol, broker, runtime, exchange, strategy.
|
|
2
|
+
|
|
3
|
+
:mod:`mftik.strategy` is what a strategy is written against, and it is here
|
|
4
|
+
rather than in the STS app so it installs beside a strategy on a developer's
|
|
5
|
+
machine. Nothing in it needs a database or a running node.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from mftik.runtime import configure_logging, run_heartbeat_service
|
|
9
|
+
|
|
10
|
+
__all__ = ["configure_logging", "run_heartbeat_service"]
|
mftik/broker/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Async Redis broker — pub/sub, request-reply, and bidirectional streams."""
|
|
2
|
+
|
|
3
|
+
from mftik.broker.client import Broker, BrokerClient
|
|
4
|
+
from mftik.broker.config import BrokerConfig
|
|
5
|
+
from mftik.broker.errors import (
|
|
6
|
+
BrokerError,
|
|
7
|
+
BrokerNotConnectedError,
|
|
8
|
+
RequestTimeoutError,
|
|
9
|
+
)
|
|
10
|
+
from mftik.broker.request import IncomingRequest
|
|
11
|
+
from mftik.broker.stream import BidirectionalStream
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"BidirectionalStream",
|
|
15
|
+
"Broker",
|
|
16
|
+
"BrokerClient",
|
|
17
|
+
"BrokerConfig",
|
|
18
|
+
"BrokerError",
|
|
19
|
+
"BrokerNotConnectedError",
|
|
20
|
+
"IncomingRequest",
|
|
21
|
+
"RequestTimeoutError",
|
|
22
|
+
]
|
mftik/broker/client.py
ADDED
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
"""Async Redis broker — pub/sub and request-reply IPC."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import redis.asyncio as redis
|
|
13
|
+
from pydantic import BaseModel
|
|
14
|
+
|
|
15
|
+
from mftik.broker.config import BrokerConfig
|
|
16
|
+
from mftik.broker.errors import BrokerNotConnectedError, RequestTimeoutError
|
|
17
|
+
from mftik.broker.request import IncomingRequest
|
|
18
|
+
from mftik.broker.stream import BidirectionalStream
|
|
19
|
+
from mftik.protocol import (
|
|
20
|
+
Envelope,
|
|
21
|
+
Heartbeat,
|
|
22
|
+
HeartbeatEnvelope,
|
|
23
|
+
Topics,
|
|
24
|
+
UntypedEnvelope,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
Handler = Callable[[IncomingRequest], Awaitable[None]]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _to_json(value: BaseModel | dict[str, Any]) -> str:
|
|
33
|
+
if isinstance(value, BaseModel):
|
|
34
|
+
return value.model_dump_json()
|
|
35
|
+
return json.dumps(value, default=str)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Broker:
|
|
39
|
+
"""Async Redis IPC client.
|
|
40
|
+
|
|
41
|
+
Three primitives:
|
|
42
|
+
|
|
43
|
+
1. **Pub/Sub** — fan-out broadcast via Redis Pub/Sub
|
|
44
|
+
(``publish`` / ``subscribe``).
|
|
45
|
+
2. **Request-reply** — 1:1 RPC via Redis lists
|
|
46
|
+
(``request`` / ``serve``).
|
|
47
|
+
3. **Bidirectional stream** — duplex channel = pub + sub
|
|
48
|
+
(``bistream``).
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
config: BrokerConfig | None = None,
|
|
54
|
+
*,
|
|
55
|
+
redis_client: redis.Redis | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
self.config = config or BrokerConfig.from_env()
|
|
58
|
+
self._redis = redis_client
|
|
59
|
+
self._owns_redis = redis_client is None
|
|
60
|
+
|
|
61
|
+
# --- lifecycle ---------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def redis(self) -> redis.Redis:
|
|
65
|
+
if self._redis is None:
|
|
66
|
+
raise BrokerNotConnectedError(
|
|
67
|
+
"Broker is not connected; call connect() first"
|
|
68
|
+
)
|
|
69
|
+
return self._redis
|
|
70
|
+
|
|
71
|
+
async def connect(self) -> None:
|
|
72
|
+
if self._redis is None:
|
|
73
|
+
self._redis = redis.from_url(
|
|
74
|
+
self.config.redis_url,
|
|
75
|
+
decode_responses=True,
|
|
76
|
+
# Pooled connections are handed out newest-first, so one that
|
|
77
|
+
# sinks to the bottom of the pool can idle past the server's
|
|
78
|
+
# ``timeout`` and be closed there. Nothing notices until it is
|
|
79
|
+
# borrowed again, and then the command fails on a socket that
|
|
80
|
+
# was already gone — which is how a domain gets a burst of
|
|
81
|
+
# ConnectionErrors on a Redis that is perfectly healthy.
|
|
82
|
+
# Checking a connection's health on checkout retires those
|
|
83
|
+
# before a caller can trip over one.
|
|
84
|
+
health_check_interval=self.config.health_check_interval,
|
|
85
|
+
socket_keepalive=True,
|
|
86
|
+
)
|
|
87
|
+
self._owns_redis = True
|
|
88
|
+
await self._redis.ping()
|
|
89
|
+
logger.info("Connected to Redis at %s", self.config.redis_url)
|
|
90
|
+
|
|
91
|
+
async def close(self) -> None:
|
|
92
|
+
if self._redis is not None and self._owns_redis:
|
|
93
|
+
await self._redis.aclose()
|
|
94
|
+
self._redis = None
|
|
95
|
+
|
|
96
|
+
async def __aenter__(self) -> Broker:
|
|
97
|
+
await self.connect()
|
|
98
|
+
return self
|
|
99
|
+
|
|
100
|
+
async def __aexit__(self, *args: object) -> None:
|
|
101
|
+
await self.close()
|
|
102
|
+
|
|
103
|
+
# --- key helpers -------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
def _rpc_queue(self, subject: str) -> str:
|
|
106
|
+
return f"{self.config.key_prefix}:rpc:{subject}"
|
|
107
|
+
|
|
108
|
+
def _rpc_reply(self, request_id: str) -> str:
|
|
109
|
+
return f"{self.config.key_prefix}:rpc:reply:{request_id}"
|
|
110
|
+
|
|
111
|
+
def _log_buffer_key(self, topic: str) -> str:
|
|
112
|
+
return f"{self.config.key_prefix}:logbuf:{topic}"
|
|
113
|
+
|
|
114
|
+
def state_key(self, name: str) -> str:
|
|
115
|
+
"""Redis key backing a shared state hash (e.g. ``td.ledger.7``)."""
|
|
116
|
+
return f"{self.config.key_prefix}:state:{name}"
|
|
117
|
+
|
|
118
|
+
# --- shared state (hashes) ---------------------------------------------
|
|
119
|
+
#
|
|
120
|
+
# Pub/Sub tells a reader that something changed; these hold what it
|
|
121
|
+
# changed *to*. A late subscriber, a restarted process and a strategy that
|
|
122
|
+
# missed a message all read the same current answer here, which is what
|
|
123
|
+
# makes "the writer's state and the reader's state agree" true by
|
|
124
|
+
# construction rather than by both sides keeping their own copy in sync.
|
|
125
|
+
|
|
126
|
+
async def state_put(
|
|
127
|
+
self, name: str, field: str, value: BaseModel | dict[str, Any]
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Write one field of a state hash."""
|
|
130
|
+
await self.redis.hset( # type: ignore[misc]
|
|
131
|
+
self.state_key(name), field, _to_json(value)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
async def state_put_many(
|
|
135
|
+
self, name: str, values: Mapping[str, BaseModel | dict[str, Any]]
|
|
136
|
+
) -> None:
|
|
137
|
+
"""Write several fields in one round trip."""
|
|
138
|
+
if not values:
|
|
139
|
+
return
|
|
140
|
+
await self.redis.hset( # type: ignore[misc]
|
|
141
|
+
self.state_key(name),
|
|
142
|
+
mapping={k: _to_json(v) for k, v in values.items()},
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
async def state_replace(
|
|
146
|
+
self, name: str, values: Mapping[str, BaseModel | dict[str, Any]]
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Make the hash exactly ``values`` — the recon path.
|
|
149
|
+
|
|
150
|
+
Delete and rewrite run in one transaction so a reader never observes
|
|
151
|
+
the empty gap between them.
|
|
152
|
+
"""
|
|
153
|
+
key = self.state_key(name)
|
|
154
|
+
pipe = self.redis.pipeline(transaction=True)
|
|
155
|
+
pipe.delete(key)
|
|
156
|
+
if values:
|
|
157
|
+
pipe.hset(key, mapping={k: _to_json(v) for k, v in values.items()})
|
|
158
|
+
await pipe.execute()
|
|
159
|
+
|
|
160
|
+
async def state_get(self, name: str, field: str) -> dict[str, Any] | None:
|
|
161
|
+
raw = await self.redis.hget(self.state_key(name), field) # type: ignore[misc]
|
|
162
|
+
return None if raw is None else json.loads(raw)
|
|
163
|
+
|
|
164
|
+
async def state_all(self, name: str) -> dict[str, dict[str, Any]]:
|
|
165
|
+
rows = await self.redis.hgetall(self.state_key(name)) # type: ignore[misc]
|
|
166
|
+
return {field: json.loads(raw) for field, raw in rows.items()}
|
|
167
|
+
|
|
168
|
+
async def state_drop(self, name: str, *fields: str) -> int:
|
|
169
|
+
if not fields:
|
|
170
|
+
return 0
|
|
171
|
+
return int(
|
|
172
|
+
await self.redis.hdel(self.state_key(name), *fields) # type: ignore[misc]
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
async def state_clear(self, *names: str) -> None:
|
|
176
|
+
"""Delete whole state hashes — call when their owner goes away.
|
|
177
|
+
|
|
178
|
+
State that outlives its writer is worse than no state: a reader cannot
|
|
179
|
+
tell a stale answer from a current one.
|
|
180
|
+
"""
|
|
181
|
+
if names:
|
|
182
|
+
await self.redis.delete(*(self.state_key(n) for n in names))
|
|
183
|
+
|
|
184
|
+
# --- recorded tape (streams) -------------------------------------------
|
|
185
|
+
#
|
|
186
|
+
# A feed's own history, kept so a strategy that starts later can warm up on
|
|
187
|
+
# what it missed. Streams rather than lists because the retention policy is
|
|
188
|
+
# a *duration* — a stream id is a millisecond timestamp, so "keep two
|
|
189
|
+
# hours" is ``XTRIM MINID`` and "read from T" is ``XRANGE``, neither of
|
|
190
|
+
# which a list can express: ``LTRIM`` counts entries, and the same count is
|
|
191
|
+
# eight hours of a quiet instrument or twenty minutes of a busy one.
|
|
192
|
+
#
|
|
193
|
+
# Two bounds, and they mean different things. ``maxlen`` on append is the
|
|
194
|
+
# memory fuse — approximate, so Redis trims whole nodes and the write stays
|
|
195
|
+
# cheap. The MINID trim is the intent. Whichever binds first is what the
|
|
196
|
+
# reader gets, and :meth:`tape_coverage` is how it finds out which.
|
|
197
|
+
|
|
198
|
+
def tape_key(self, feed: str) -> str:
|
|
199
|
+
"""Redis stream holding recorded tape for ``feed``."""
|
|
200
|
+
return f"{self.config.key_prefix}:tape:{feed}"
|
|
201
|
+
|
|
202
|
+
def tape_coverage_key(self, feed: str) -> str:
|
|
203
|
+
"""Redis hash describing what :meth:`tape_key` currently covers."""
|
|
204
|
+
return f"{self.config.key_prefix}:tape:coverage:{feed}"
|
|
205
|
+
|
|
206
|
+
async def tape_append(
|
|
207
|
+
self,
|
|
208
|
+
feed: str,
|
|
209
|
+
fields: Mapping[str, str],
|
|
210
|
+
*,
|
|
211
|
+
maxlen: int,
|
|
212
|
+
ttl_seconds: int,
|
|
213
|
+
) -> None:
|
|
214
|
+
"""Append one record, capping the stream at ``maxlen`` entries.
|
|
215
|
+
|
|
216
|
+
The id is Redis' own clock, not the venue's timestamp. Event time is a
|
|
217
|
+
field on the record instead, because ``XADD`` refuses an id that does
|
|
218
|
+
not exceed the last one and a venue tape is not strictly monotonic —
|
|
219
|
+
one late print out of a million would otherwise end the recording.
|
|
220
|
+
|
|
221
|
+
``ttl_seconds`` is renewed on every append, so a feed that stops being
|
|
222
|
+
recorded expires on its own. Without it a tape would outlive the last
|
|
223
|
+
strategy that ever wanted it: the MINID trim only runs against feeds
|
|
224
|
+
that are still pumping, and a stream nobody writes to is never capped
|
|
225
|
+
by ``maxlen`` either. Every instrument ever subscribed would keep its
|
|
226
|
+
last two hours for as long as Redis lived.
|
|
227
|
+
"""
|
|
228
|
+
pipe = self.redis.pipeline()
|
|
229
|
+
pipe.xadd(
|
|
230
|
+
self.tape_key(feed),
|
|
231
|
+
dict(fields),
|
|
232
|
+
maxlen=maxlen,
|
|
233
|
+
approximate=True,
|
|
234
|
+
)
|
|
235
|
+
pipe.expire(self.tape_key(feed), ttl_seconds)
|
|
236
|
+
pipe.expire(self.tape_coverage_key(feed), ttl_seconds)
|
|
237
|
+
await pipe.execute()
|
|
238
|
+
|
|
239
|
+
async def tape_tail(
|
|
240
|
+
self, feed: str, *, count: int
|
|
241
|
+
) -> list[tuple[str, dict[str, str]]]:
|
|
242
|
+
"""Read the newest ``count`` records, oldest → newest.
|
|
243
|
+
|
|
244
|
+
The newest rather than the oldest: warming up means catching up to now,
|
|
245
|
+
and a stream capped by two independent bounds holds an unknown number
|
|
246
|
+
of records, so "the first N" is not a window anyone asked for.
|
|
247
|
+
"""
|
|
248
|
+
if count <= 0:
|
|
249
|
+
return []
|
|
250
|
+
rows = await self.redis.xrevrange(
|
|
251
|
+
self.tape_key(feed), max="+", min="-", count=count
|
|
252
|
+
)
|
|
253
|
+
return [(str(rid), dict(fields)) for rid, fields in reversed(rows)]
|
|
254
|
+
|
|
255
|
+
async def tape_trim_before(self, feed: str, *, min_id_ms: int) -> int:
|
|
256
|
+
"""Drop records older than ``min_id_ms``. Returns how many went."""
|
|
257
|
+
return int(
|
|
258
|
+
await self.redis.xtrim(self.tape_key(feed), minid=min_id_ms)
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
async def tape_mark_recording(
|
|
262
|
+
self, feed: str, *, since_ms: int, ttl_seconds: int
|
|
263
|
+
) -> None:
|
|
264
|
+
"""Record that this feed started recording at ``since_ms``.
|
|
265
|
+
|
|
266
|
+
Called when a feed begins pumping, which is also the moment continuity
|
|
267
|
+
breaks: whatever is already in the stream predates a gap of unknown
|
|
268
|
+
length. Readers compare against this rather than assuming the records
|
|
269
|
+
they can see form one series.
|
|
270
|
+
|
|
271
|
+
Carries its own TTL because a feed can be subscribed and then print
|
|
272
|
+
nothing at all — a dead instrument, a venue outage — and the appends
|
|
273
|
+
that would otherwise renew it never come.
|
|
274
|
+
"""
|
|
275
|
+
pipe = self.redis.pipeline()
|
|
276
|
+
pipe.hset(
|
|
277
|
+
self.tape_coverage_key(feed),
|
|
278
|
+
mapping={
|
|
279
|
+
"continuous_since_ms": str(since_ms),
|
|
280
|
+
"recording": "1",
|
|
281
|
+
"stopped_ms": "",
|
|
282
|
+
},
|
|
283
|
+
)
|
|
284
|
+
pipe.expire(self.tape_coverage_key(feed), ttl_seconds)
|
|
285
|
+
await pipe.execute()
|
|
286
|
+
|
|
287
|
+
async def tape_mark_stopped(self, feed: str, *, at_ms: int) -> None:
|
|
288
|
+
"""Record that this feed stopped recording at ``at_ms``.
|
|
289
|
+
|
|
290
|
+
The stream is left alone. A reader that wants the last two hours before
|
|
291
|
+
a feed went quiet can still have them — it just has to know they end,
|
|
292
|
+
and that is exactly what this says.
|
|
293
|
+
"""
|
|
294
|
+
await self.redis.hset( # type: ignore[misc]
|
|
295
|
+
self.tape_coverage_key(feed),
|
|
296
|
+
mapping={"recording": "0", "stopped_ms": str(at_ms)},
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
async def tape_coverage(self, feed: str) -> dict[str, str]:
|
|
300
|
+
"""What :meth:`tape_key` covers, or ``{}`` if it was never recorded."""
|
|
301
|
+
return dict(await self.redis.hgetall(self.tape_coverage_key(feed))) # type: ignore[misc]
|
|
302
|
+
|
|
303
|
+
# --- Pub/Sub -----------------------------------------------------------
|
|
304
|
+
|
|
305
|
+
async def publish(self, topic: str, envelope: Envelope[Any]) -> int:
|
|
306
|
+
"""Publish an envelope to a pub/sub topic (fan-out)."""
|
|
307
|
+
return int(await self.redis.publish(topic, envelope.to_json()))
|
|
308
|
+
|
|
309
|
+
async def publish_log(
|
|
310
|
+
self,
|
|
311
|
+
topic: str,
|
|
312
|
+
envelope: Envelope[Any],
|
|
313
|
+
*,
|
|
314
|
+
maxlen: int | None = None,
|
|
315
|
+
ttl_seconds: int = 86_400,
|
|
316
|
+
) -> int:
|
|
317
|
+
"""Publish a log line and append it to a Redis list for late subscribers.
|
|
318
|
+
|
|
319
|
+
Redis Pub/Sub alone drops messages when nobody is listening (e.g. UI
|
|
320
|
+
opens ``/ws/sts/...`` after deploy). The buffer is replayed on connect.
|
|
321
|
+
``maxlen`` defaults to :attr:`BrokerConfig.log_buffer_maxlen` (100).
|
|
322
|
+
"""
|
|
323
|
+
keep = (
|
|
324
|
+
self.config.log_buffer_maxlen if maxlen is None else max(1, maxlen)
|
|
325
|
+
)
|
|
326
|
+
raw = envelope.to_json()
|
|
327
|
+
key = self._log_buffer_key(topic)
|
|
328
|
+
pipe = self.redis.pipeline()
|
|
329
|
+
pipe.rpush(key, raw)
|
|
330
|
+
pipe.ltrim(key, -keep, -1)
|
|
331
|
+
pipe.expire(key, ttl_seconds)
|
|
332
|
+
pipe.publish(topic, raw)
|
|
333
|
+
results = await pipe.execute()
|
|
334
|
+
return int(results[-1])
|
|
335
|
+
|
|
336
|
+
async def fetch_log_buffer(self, topic: str) -> list[str]:
|
|
337
|
+
"""Return buffered log JSON lines for ``topic`` (oldest → newest)."""
|
|
338
|
+
rows = await self.redis.lrange(self._log_buffer_key(topic), 0, -1)
|
|
339
|
+
return list(rows)
|
|
340
|
+
|
|
341
|
+
async def subscribe(
|
|
342
|
+
self,
|
|
343
|
+
topics: str | Sequence[str],
|
|
344
|
+
*,
|
|
345
|
+
stop: asyncio.Event | None = None,
|
|
346
|
+
) -> AsyncIterator[UntypedEnvelope]:
|
|
347
|
+
"""Yield envelopes from one or more pub/sub topics until ``stop``.
|
|
348
|
+
|
|
349
|
+
Uses Redis Pub/Sub. Messages published while not subscribed are lost
|
|
350
|
+
unless they were also written via :meth:`publish_log`.
|
|
351
|
+
"""
|
|
352
|
+
channel_list = (topics,) if isinstance(topics, str) else tuple(topics)
|
|
353
|
+
if not channel_list:
|
|
354
|
+
raise ValueError("subscribe requires at least one topic")
|
|
355
|
+
|
|
356
|
+
pubsub = self.redis.pubsub()
|
|
357
|
+
await pubsub.subscribe(*channel_list)
|
|
358
|
+
try:
|
|
359
|
+
while stop is None or not stop.is_set():
|
|
360
|
+
message = await pubsub.get_message(
|
|
361
|
+
ignore_subscribe_messages=True, timeout=1.0
|
|
362
|
+
)
|
|
363
|
+
if message is None:
|
|
364
|
+
await asyncio.sleep(0.01)
|
|
365
|
+
continue
|
|
366
|
+
data = message.get("data")
|
|
367
|
+
if data is None:
|
|
368
|
+
continue
|
|
369
|
+
yield UntypedEnvelope.from_json(data)
|
|
370
|
+
finally:
|
|
371
|
+
await pubsub.unsubscribe(*channel_list)
|
|
372
|
+
await pubsub.aclose()
|
|
373
|
+
|
|
374
|
+
async def psubscribe(
|
|
375
|
+
self,
|
|
376
|
+
patterns: str | Sequence[str],
|
|
377
|
+
*,
|
|
378
|
+
stop: asyncio.Event | None = None,
|
|
379
|
+
) -> AsyncIterator[tuple[str, UntypedEnvelope]]:
|
|
380
|
+
"""Yield ``(channel, envelope)`` from pattern subscriptions until ``stop``.
|
|
381
|
+
|
|
382
|
+
Uses Redis ``PSUBSCRIBE``. Messages published while not subscribed are
|
|
383
|
+
lost unless they were also written via :meth:`publish_log`.
|
|
384
|
+
"""
|
|
385
|
+
pattern_list = (patterns,) if isinstance(patterns, str) else tuple(patterns)
|
|
386
|
+
if not pattern_list:
|
|
387
|
+
raise ValueError("psubscribe requires at least one pattern")
|
|
388
|
+
|
|
389
|
+
pubsub = self.redis.pubsub()
|
|
390
|
+
await pubsub.psubscribe(*pattern_list)
|
|
391
|
+
try:
|
|
392
|
+
while stop is None or not stop.is_set():
|
|
393
|
+
message = await pubsub.get_message(
|
|
394
|
+
ignore_subscribe_messages=True, timeout=1.0
|
|
395
|
+
)
|
|
396
|
+
if message is None:
|
|
397
|
+
await asyncio.sleep(0.01)
|
|
398
|
+
continue
|
|
399
|
+
if message.get("type") != "pmessage":
|
|
400
|
+
continue
|
|
401
|
+
data = message.get("data")
|
|
402
|
+
channel = message.get("channel")
|
|
403
|
+
if data is None or channel is None:
|
|
404
|
+
continue
|
|
405
|
+
yield str(channel), UntypedEnvelope.from_json(data)
|
|
406
|
+
finally:
|
|
407
|
+
await pubsub.punsubscribe(*pattern_list)
|
|
408
|
+
await pubsub.aclose()
|
|
409
|
+
|
|
410
|
+
def bistream(
|
|
411
|
+
self,
|
|
412
|
+
*,
|
|
413
|
+
tx: str,
|
|
414
|
+
rx: str,
|
|
415
|
+
) -> BidirectionalStream:
|
|
416
|
+
"""Open a bidirectional stream (publish on ``tx``, subscribe on ``rx``)."""
|
|
417
|
+
return BidirectionalStream(self, tx=tx, rx=rx)
|
|
418
|
+
|
|
419
|
+
def bistream_pair(
|
|
420
|
+
self,
|
|
421
|
+
name: str,
|
|
422
|
+
) -> tuple[BidirectionalStream, BidirectionalStream]:
|
|
423
|
+
"""Open both ends of a named bistream: ``(up, down)``.
|
|
424
|
+
|
|
425
|
+
``up`` publishes ``bistream.{name}.up`` and receives ``.down``;
|
|
426
|
+
``down`` is the complement.
|
|
427
|
+
"""
|
|
428
|
+
up_topic, down_topic = BidirectionalStream.topics(name)
|
|
429
|
+
up = self.bistream(tx=up_topic, rx=down_topic)
|
|
430
|
+
down = self.bistream(tx=down_topic, rx=up_topic)
|
|
431
|
+
return up, down
|
|
432
|
+
|
|
433
|
+
# --- Request-reply -----------------------------------------------------
|
|
434
|
+
|
|
435
|
+
async def request(
|
|
436
|
+
self,
|
|
437
|
+
subject: str,
|
|
438
|
+
envelope: Envelope[Any],
|
|
439
|
+
*,
|
|
440
|
+
timeout: float | None = None,
|
|
441
|
+
) -> UntypedEnvelope:
|
|
442
|
+
"""Send a request and wait for a single reply.
|
|
443
|
+
|
|
444
|
+
The envelope's ``id`` is used as the correlation id. A temporary
|
|
445
|
+
reply list key is written into ``reply_to`` before enqueueing.
|
|
446
|
+
"""
|
|
447
|
+
wait = self.config.request_timeout if timeout is None else timeout
|
|
448
|
+
reply_key = self._rpc_reply(envelope.id)
|
|
449
|
+
outbound = (
|
|
450
|
+
envelope
|
|
451
|
+
if envelope.reply_to == reply_key
|
|
452
|
+
else envelope.model_copy(update={"reply_to": reply_key})
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
queue = self._rpc_queue(subject)
|
|
456
|
+
await self.redis.rpush(queue, outbound.to_json())
|
|
457
|
+
|
|
458
|
+
# BLPOP timeout is whole seconds; poll until the deadline for accuracy.
|
|
459
|
+
deadline = time.monotonic() + wait
|
|
460
|
+
try:
|
|
461
|
+
while True:
|
|
462
|
+
remaining = deadline - time.monotonic()
|
|
463
|
+
if remaining <= 0:
|
|
464
|
+
raise RequestTimeoutError(subject, outbound.id, wait)
|
|
465
|
+
result = await self.redis.blpop(reply_key, timeout=1)
|
|
466
|
+
if result is None:
|
|
467
|
+
continue
|
|
468
|
+
_key, data = result
|
|
469
|
+
return UntypedEnvelope.from_json(data)
|
|
470
|
+
finally:
|
|
471
|
+
await self.redis.delete(reply_key)
|
|
472
|
+
|
|
473
|
+
async def post(self, subject: str, envelope: Envelope[Any]) -> None:
|
|
474
|
+
"""Enqueue on a request-reply subject without waiting for a reply.
|
|
475
|
+
|
|
476
|
+
The same queue :meth:`request` uses and the same competing consumers
|
|
477
|
+
take from it; what is missing is the ``reply_to``, so the handler
|
|
478
|
+
answers nobody and this returns as soon as Redis has the message.
|
|
479
|
+
|
|
480
|
+
For work whose *result* the sender has no use for and whose duration it
|
|
481
|
+
must not inherit — a backfill run is minutes of venue round trips, and
|
|
482
|
+
the shutdown path that asks for one is measured in seconds. A request
|
|
483
|
+
left in the list because nothing is serving the subject yet is not lost:
|
|
484
|
+
the next consumer to come up takes it, which is the recovery a pub/sub
|
|
485
|
+
message could not offer.
|
|
486
|
+
"""
|
|
487
|
+
await self.redis.rpush(self._rpc_queue(subject), envelope.to_json())
|
|
488
|
+
|
|
489
|
+
async def serve(
|
|
490
|
+
self,
|
|
491
|
+
subject: str,
|
|
492
|
+
*,
|
|
493
|
+
stop: asyncio.Event | None = None,
|
|
494
|
+
) -> AsyncIterator[IncomingRequest]:
|
|
495
|
+
"""Yield incoming requests on a request-reply subject.
|
|
496
|
+
|
|
497
|
+
Call ``await req.reply(envelope)`` to respond. Competing consumers
|
|
498
|
+
on the same subject share work via Redis list ``BLPOP``.
|
|
499
|
+
"""
|
|
500
|
+
queue = self._rpc_queue(subject)
|
|
501
|
+
while stop is None or not stop.is_set():
|
|
502
|
+
result = await self.redis.blpop(queue, timeout=1)
|
|
503
|
+
if result is None:
|
|
504
|
+
continue
|
|
505
|
+
_key, data = result
|
|
506
|
+
envelope = UntypedEnvelope.from_json(data)
|
|
507
|
+
yield IncomingRequest(self, envelope)
|
|
508
|
+
|
|
509
|
+
async def serve_handler(
|
|
510
|
+
self,
|
|
511
|
+
subject: str,
|
|
512
|
+
handler: Handler,
|
|
513
|
+
*,
|
|
514
|
+
stop: asyncio.Event | None = None,
|
|
515
|
+
) -> None:
|
|
516
|
+
"""Run ``handler`` for each incoming request until ``stop``."""
|
|
517
|
+
async for req in self.serve(subject, stop=stop):
|
|
518
|
+
await handler(req)
|
|
519
|
+
|
|
520
|
+
async def _send_reply(self, reply_to: str, envelope: Envelope[Any]) -> None:
|
|
521
|
+
await self.redis.rpush(reply_to, envelope.to_json())
|
|
522
|
+
await self.redis.expire(reply_to, self.config.reply_ttl_seconds)
|
|
523
|
+
|
|
524
|
+
# --- convenience -------------------------------------------------------
|
|
525
|
+
|
|
526
|
+
async def heartbeat_loop(
|
|
527
|
+
self,
|
|
528
|
+
source: str,
|
|
529
|
+
*,
|
|
530
|
+
interval: float = 5.0,
|
|
531
|
+
stop: asyncio.Event | None = None,
|
|
532
|
+
on_tick: Callable[[], None] | None = None,
|
|
533
|
+
) -> None:
|
|
534
|
+
"""Publish periodic heartbeats on the heartbeat pub/sub topic."""
|
|
535
|
+
while stop is None or not stop.is_set():
|
|
536
|
+
envelope = HeartbeatEnvelope.wrap(
|
|
537
|
+
Heartbeat(),
|
|
538
|
+
type="heartbeat",
|
|
539
|
+
source=source,
|
|
540
|
+
)
|
|
541
|
+
await self.publish(Topics.HEARTBEAT, envelope)
|
|
542
|
+
if on_tick is not None:
|
|
543
|
+
on_tick()
|
|
544
|
+
try:
|
|
545
|
+
if stop is not None:
|
|
546
|
+
await asyncio.wait_for(stop.wait(), timeout=interval)
|
|
547
|
+
else:
|
|
548
|
+
await asyncio.sleep(interval)
|
|
549
|
+
except TimeoutError:
|
|
550
|
+
continue
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
# Back-compat alias used during the rename.
|
|
554
|
+
BrokerClient = Broker
|
mftik/broker/config.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class BrokerConfig:
|
|
9
|
+
"""Redis broker connection and IPC defaults."""
|
|
10
|
+
|
|
11
|
+
redis_url: str = "redis://localhost:6379/0"
|
|
12
|
+
key_prefix: str = "mft"
|
|
13
|
+
request_timeout: float = 5.0
|
|
14
|
+
reply_ttl_seconds: int = 60
|
|
15
|
+
#: How many log lines ``publish_log`` keeps per topic for late WS
|
|
16
|
+
#: subscribers. Older lines are trimmed; live pub/sub is unaffected.
|
|
17
|
+
log_buffer_maxlen: int = 100
|
|
18
|
+
#: How old a pooled connection may be before it is pinged on checkout.
|
|
19
|
+
#: Must stay under the Redis server's ``timeout`` (300s in production) —
|
|
20
|
+
#: the point is to retire a connection the server has already closed
|
|
21
|
+
#: before a caller borrows it and fails on it.
|
|
22
|
+
health_check_interval: int = 30
|
|
23
|
+
|
|
24
|
+
@classmethod
|
|
25
|
+
def from_env(cls) -> BrokerConfig:
|
|
26
|
+
return cls(
|
|
27
|
+
redis_url=os.getenv("REDIS_URL", "redis://localhost:6379/0"),
|
|
28
|
+
key_prefix=os.getenv("BROKER_KEY_PREFIX", "mft"),
|
|
29
|
+
request_timeout=float(os.getenv("BROKER_REQUEST_TIMEOUT", "5")),
|
|
30
|
+
reply_ttl_seconds=int(os.getenv("BROKER_REPLY_TTL", "60")),
|
|
31
|
+
log_buffer_maxlen=max(
|
|
32
|
+
1, int(os.getenv("BROKER_LOG_BUFFER_MAXLEN", "100"))
|
|
33
|
+
),
|
|
34
|
+
health_check_interval=int(
|
|
35
|
+
os.getenv("BROKER_HEALTH_CHECK_INTERVAL", "30")
|
|
36
|
+
),
|
|
37
|
+
)
|
mftik/broker/errors.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Broker IPC errors."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BrokerError(Exception):
|
|
5
|
+
"""Base error for broker operations."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BrokerNotConnectedError(BrokerError):
|
|
9
|
+
"""Raised when an operation is attempted before connect()."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RequestTimeoutError(BrokerError):
|
|
13
|
+
"""Raised when a request-reply call exceeds its timeout."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, subject: str, request_id: str, timeout: float) -> None:
|
|
16
|
+
self.subject = subject
|
|
17
|
+
self.request_id = request_id
|
|
18
|
+
self.timeout = timeout
|
|
19
|
+
super().__init__(
|
|
20
|
+
f"request to {subject!r} timed out after {timeout}s (id={request_id})"
|
|
21
|
+
)
|
mftik/broker/request.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Request-reply request handle."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from mftik.protocol import Envelope, UntypedEnvelope
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from mftik.broker.client import Broker
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class IncomingRequest:
|
|
14
|
+
"""A request waiting for a reply on a request-reply subject."""
|
|
15
|
+
|
|
16
|
+
__slots__ = ("envelope", "_broker", "_replied")
|
|
17
|
+
|
|
18
|
+
def __init__(self, broker: Broker, envelope: UntypedEnvelope) -> None:
|
|
19
|
+
self.envelope = envelope
|
|
20
|
+
self._broker = broker
|
|
21
|
+
self._replied = False
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def replied(self) -> bool:
|
|
25
|
+
return self._replied
|
|
26
|
+
|
|
27
|
+
async def reply(self, envelope: Envelope[Any]) -> None:
|
|
28
|
+
"""Send a reply envelope to the requester's reply inbox, if there is one.
|
|
29
|
+
|
|
30
|
+
A missing ``reply_to`` is not an error. It is what
|
|
31
|
+
:meth:`~mftik.broker.client.Broker.post` produces — the same queue and
|
|
32
|
+
the same handlers as :meth:`~mftik.broker.client.Broker.request`, minus
|
|
33
|
+
anybody waiting — so a handler that always replies is exactly what
|
|
34
|
+
makes a subject postable. Raising here would have meant every such
|
|
35
|
+
handler needed a guard, and forgetting one would surface only on the
|
|
36
|
+
posted path, after the work was already done.
|
|
37
|
+
|
|
38
|
+
:attr:`replied` distinguishes the two afterwards: a handler that wants
|
|
39
|
+
to skip building an answer nobody will read can check ``reply_to``
|
|
40
|
+
itself first.
|
|
41
|
+
"""
|
|
42
|
+
reply_to = self.envelope.reply_to
|
|
43
|
+
if not reply_to:
|
|
44
|
+
return
|
|
45
|
+
await self._broker._send_reply(reply_to, envelope)
|
|
46
|
+
self._replied = True
|