serverish 1.1.7__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {serverish-1.1.7 → serverish-1.2.0}/PKG-INFO +1 -1
- {serverish-1.1.7 → serverish-1.2.0}/pyproject.toml +1 -1
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_reader.py +232 -175
- {serverish-1.1.7 → serverish-1.2.0}/LICENSE +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/README.md +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/__init__.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/__init__.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/asyncio_util_functions.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/collector.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/datetime.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/exceptions.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/fifoset.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/hasstatuses.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/idmanger.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/manageable.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/singleton.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/status.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/base/task_manager.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/connection/__init__.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/connection/connection.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/connection/connection_jets.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/connection/connection_nats.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/__init__.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/logging.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/messenger.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_callback_sub.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_journal_pub.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_journal_read.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_progress_pub.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_progress_read.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_publisher.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_rpc_req.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_rpc_resp.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_single_pub.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msg_single_read.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/messenger/msgvalidator.py +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/config.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/default.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/include/timestamp.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/journal.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/meta.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/program_current_state.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/program_state.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/progress.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/schema/telemetry.schema.json +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/services/TODO.txt +0 -0
- {serverish-1.1.7 → serverish-1.2.0}/serverish/slog.py +0 -0
|
@@ -5,6 +5,9 @@ import asyncio
|
|
|
5
5
|
from asyncio import Event
|
|
6
6
|
from collections import deque
|
|
7
7
|
from datetime import datetime
|
|
8
|
+
import time
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
import functools
|
|
8
11
|
|
|
9
12
|
import nats.errors
|
|
10
13
|
import param
|
|
@@ -42,8 +45,8 @@ class MsgReader(MsgDriver):
|
|
|
42
45
|
doc="Additional JetStream consumer configuration")
|
|
43
46
|
|
|
44
47
|
# behaviour config
|
|
45
|
-
|
|
46
|
-
|
|
48
|
+
error_behavior = param.ObjectSelector(default="WAIT", objects=["RAISE", "FINISH", "WAIT"],
|
|
49
|
+
doc="On disconnection: "
|
|
47
50
|
"RAISE - reraise exception, "
|
|
48
51
|
"FINISH - silently finish iteration, "
|
|
49
52
|
"WAIT - wait for connection to be re-established")
|
|
@@ -59,8 +62,13 @@ class MsgReader(MsgDriver):
|
|
|
59
62
|
**kwargs) -> None:
|
|
60
63
|
if parent is None:
|
|
61
64
|
parent = Messenger()
|
|
62
|
-
|
|
63
|
-
|
|
65
|
+
consumer_cfg_defaults = {
|
|
66
|
+
'inactive_threshold': 60
|
|
67
|
+
}
|
|
68
|
+
if consumer_cfg is not None:
|
|
69
|
+
consumer_cfg_defaults.update(consumer_cfg)
|
|
70
|
+
self.batch = 100
|
|
71
|
+
self.messages = deque()
|
|
64
72
|
self.pull_subscription: JetStreamContext.PullSubscription | None = None
|
|
65
73
|
self.push_subscription: JetStreamContext.PushSubscription | None = None
|
|
66
74
|
self.last_seq: int | None = None
|
|
@@ -70,11 +78,13 @@ class MsgReader(MsgDriver):
|
|
|
70
78
|
self._reconnect_needed: Event = Event()
|
|
71
79
|
self.pull_batch = deque()
|
|
72
80
|
self.id_cache = FifoSet(128)
|
|
81
|
+
self._expect_beeing_open = False
|
|
73
82
|
super().__init__(subject=subject, parent=parent,
|
|
74
|
-
deliver_policy=deliver_policy, opt_start_time=opt_start_time, consumer_cfg=
|
|
83
|
+
deliver_policy=deliver_policy, opt_start_time=opt_start_time, consumer_cfg=consumer_cfg_defaults,
|
|
75
84
|
**kwargs)
|
|
76
85
|
log.debug(f"Created {self}")
|
|
77
86
|
|
|
87
|
+
|
|
78
88
|
def __aiter__(self):
|
|
79
89
|
log.debug(f"Entering iteration {self}")
|
|
80
90
|
return self
|
|
@@ -88,184 +98,228 @@ class MsgReader(MsgDriver):
|
|
|
88
98
|
finally:
|
|
89
99
|
pass
|
|
90
100
|
|
|
91
|
-
|
|
92
101
|
async def read_next(self) -> tuple[dict, dict]:
|
|
93
|
-
|
|
94
|
-
await self.open()
|
|
102
|
+
"""By default, once you enter method will not return or raise exception until have some data for you
|
|
95
103
|
|
|
104
|
+
Only stop signal and Cancel exception finish the iteration.
|
|
105
|
+
One may change this behavior by setting `on_connection_close` to 'RAISE' or 'FINISH'.
|
|
106
|
+
This will cause raising exception or finishing iteration on seruios erros.
|
|
107
|
+
"""
|
|
96
108
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
# The reader loop's state and methods
|
|
110
|
+
@dataclass
|
|
111
|
+
class _ReadNextState:
|
|
112
|
+
class _LoopException(Exception):
|
|
113
|
+
def __init__(self, task: str, *args, **kwargs):
|
|
114
|
+
self.task: str = task
|
|
115
|
+
super().__init__(*args, **kwargs)
|
|
116
|
+
class ContinueException(_LoopException): pass
|
|
117
|
+
class ErrorException(_LoopException):
|
|
118
|
+
def __init__(self, task: str, e: Exception, *args, **kwargs):
|
|
119
|
+
self.error: Exception = e
|
|
120
|
+
super().__init__(task, *args, **kwargs)
|
|
121
|
+
class ReturnException(_LoopException):
|
|
122
|
+
def __init__(self, task: str, data: dict, meta: dict, *args, **kwargs):
|
|
123
|
+
self.data: dict = data
|
|
124
|
+
self.meta: dict = meta
|
|
125
|
+
super().__init__(task, *args, **kwargs)
|
|
126
|
+
class EndIterationException(_LoopException): pass
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
reader: MsgReader
|
|
130
|
+
n: int = 0
|
|
131
|
+
log: List[str] = field(default_factory=list)
|
|
132
|
+
start_time: datetime = field(default_factory=datetime.now)
|
|
133
|
+
error: Exception = None
|
|
134
|
+
|
|
135
|
+
def async_shield(func):
|
|
136
|
+
@functools.wraps(func)
|
|
137
|
+
async def wrapper(*args, **kwargs):
|
|
111
138
|
try:
|
|
112
|
-
await
|
|
113
|
-
|
|
139
|
+
result = await func(*args, **kwargs)
|
|
140
|
+
except _ReadNextState._LoopException:
|
|
141
|
+
raise
|
|
114
142
|
except Exception as e:
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
143
|
+
raise _ReadNextState.ErrorException(func.__name__, e)
|
|
144
|
+
return result
|
|
145
|
+
return wrapper
|
|
146
|
+
|
|
147
|
+
@async_shield
|
|
148
|
+
async def pop_msg(self) -> None:
|
|
149
|
+
if len(self.reader.messages) > 0:
|
|
150
|
+
bmsg = self.reader.messages.popleft()
|
|
151
|
+
try: # nonessential
|
|
152
|
+
await bmsg.ack()
|
|
153
|
+
except Exception as e:
|
|
154
|
+
log.warning(self.fmt(f"Error acking message: {e}"))
|
|
155
|
+
data, meta = self.reader.messenger.unpack_nats_msg(bmsg)
|
|
156
|
+
meta['receive_mode'] = 'pull'
|
|
157
|
+
try: # nonessential
|
|
158
|
+
self.reader.messenger.log_msg_trace(data, meta, f"SUB PULL iteration from {self.reader.subject}")
|
|
159
|
+
self.reader.last_seq = meta['nats']['seq']
|
|
160
|
+
if len(self.reader.messages) == 0:
|
|
161
|
+
self.reader._emptied.set()
|
|
162
|
+
except Exception as e:
|
|
163
|
+
log.warning(self.fmt(f"Error unpacking message: {e}"))
|
|
164
|
+
raise self.ReturnException('pop', data, meta)
|
|
165
|
+
@async_shield
|
|
166
|
+
async def ensure_open(self) -> None:
|
|
167
|
+
if not self.reader.is_open:
|
|
168
|
+
if self.reader._expect_beeing_open:
|
|
169
|
+
log.warning(self.fmt("The reader is not open, but once it has been, trying to reopen"))
|
|
170
|
+
await self.reader.open()
|
|
171
|
+
if self.reader.is_open:
|
|
172
|
+
self.reader._expect_beeing_open = True
|
|
173
|
+
raise self.ContinueException('open')
|
|
174
|
+
else:
|
|
175
|
+
raise Exception('Opened, but still close')
|
|
176
|
+
@async_shield
|
|
177
|
+
async def ensure_not_stopped(self) -> None:
|
|
178
|
+
if self.reader._stop.is_set():
|
|
179
|
+
await self.reader.close()
|
|
180
|
+
raise self.EndIterationException('stop')
|
|
181
|
+
@async_shield
|
|
182
|
+
async def ensure_consumer(self) -> None:
|
|
183
|
+
if self.error is not None:
|
|
184
|
+
try:
|
|
185
|
+
ci = await self.reader.pull_subscription.consumer_info()
|
|
186
|
+
print(ci)
|
|
187
|
+
except nats.js.errors.NotFoundError:
|
|
188
|
+
log.warning(self.fmt("Consumer has gone, trying to recreate it"))
|
|
189
|
+
await self.reader._reopen()
|
|
190
|
+
log.info(self.fmt(f"Consumer re-opened"))
|
|
191
|
+
raise self.ContinueException('reopen')
|
|
192
|
+
@async_shield
|
|
193
|
+
async def read_batch(self) -> None:
|
|
194
|
+
if len(self.reader.messages) == 0:
|
|
195
|
+
timeout = min(0.1 + self.n/10.0, 5.0)
|
|
196
|
+
# batch = 1 if self.error is not None else 100 # recover slowly
|
|
197
|
+
log.debug(self.fmt(f"Pulling {self.reader.batch} messages with timeout {timeout}s"))
|
|
198
|
+
new_msgs = await self.reader.fetch_available(batch=self.reader.batch, timeout=timeout)
|
|
199
|
+
log.debug(self.fmt(f"Pulled {len(new_msgs)} messages"))
|
|
200
|
+
self.reader.messages.extend(new_msgs)
|
|
201
|
+
raise self.ContinueException('read')
|
|
202
|
+
|
|
203
|
+
def fmt(self, msg: str) -> str:
|
|
204
|
+
return f"({self.n}){self.reader} {msg} elapsed: {(datetime.now() - self.start_time).total_seconds():.1f}s hist: {':'.join(self.log)}"
|
|
205
|
+
def logput(self, msg: str) -> None:
|
|
206
|
+
if len(self.log) > 0 and self.log[-1] == msg:
|
|
207
|
+
return
|
|
208
|
+
if len(self.log) > 15:
|
|
209
|
+
self.log = self.log[-15:]
|
|
210
|
+
self.log[0] = '...'
|
|
211
|
+
self.log.append(msg)
|
|
212
|
+
|
|
213
|
+
st = _ReadNextState(self)
|
|
214
|
+
# The loop which tries hard to get some data
|
|
215
|
+
while True:
|
|
216
|
+
try:
|
|
217
|
+
# 0. Do we have some data to return already?
|
|
218
|
+
await st.pop_msg()
|
|
219
|
+
# 1. Check if iteration have been stopped externally
|
|
220
|
+
await st.ensure_not_stopped()
|
|
221
|
+
# 2. Is it at least open?
|
|
222
|
+
await st.ensure_open()
|
|
223
|
+
# 3. Check consumer, maybe ephemeral consumer is gone
|
|
224
|
+
await st.ensure_consumer()
|
|
225
|
+
# 4. Pull batch of messages
|
|
226
|
+
await st.read_batch()
|
|
227
|
+
except st.ContinueException as e: # one of the method did something
|
|
228
|
+
st.logput(f'{e.task}-ok')
|
|
229
|
+
log.debug(st.fmt(f"continue after: {e.task}"))
|
|
230
|
+
except st.ReturnException as e: # we have data to return
|
|
231
|
+
st.logput(f'{e.task}-ret')
|
|
232
|
+
if st.error is not None:
|
|
233
|
+
log.info(st.fmt(f"recovered"))
|
|
234
|
+
st.error = None
|
|
235
|
+
log.debug(st.fmt(f"data returned"))
|
|
236
|
+
return e.data, e.meta
|
|
237
|
+
except st.EndIterationException as e:
|
|
238
|
+
st.logput(f'{e.task}-fin')
|
|
239
|
+
log.info(st.fmt(f"iteration stoped on request"))
|
|
240
|
+
raise MessengerReaderStopped
|
|
241
|
+
except st.ErrorException as e: # some error
|
|
242
|
+
st.logput(f'{e.task}-err')
|
|
243
|
+
st.error = e.error
|
|
244
|
+
match self.error_behavior:
|
|
245
|
+
case 'RAISE':
|
|
246
|
+
log.error(st.fmt(f"raising read_next error: {e.error}"))
|
|
247
|
+
raise e.error
|
|
248
|
+
case 'FINISH':
|
|
249
|
+
log.error(st.fmt(f"finishing iteration on error: {e.error}"))
|
|
250
|
+
raise MessengerReaderStopped
|
|
251
|
+
case 'WAIT':
|
|
252
|
+
wait_time = min(0.2 + st.n/5.0, 15.0)
|
|
253
|
+
log.warning(st.fmt(f"read_next error, (retry in {wait_time:.1f}s): {e.error}"))
|
|
254
|
+
await asyncio.sleep(wait_time)
|
|
255
|
+
case _: # should not be reached
|
|
256
|
+
log.error(st.fmt(f"Invalid on_connection_close value {self.error_behavior}"))
|
|
257
|
+
exit(-1) # this is not i/o error but programming error
|
|
258
|
+
except Exception as e:# should never happen
|
|
259
|
+
log.error(st.fmt(f"unhandled exception {e}"))
|
|
260
|
+
exit(-1) # this is not i/o error but programming error
|
|
261
|
+
st.n += 1
|
|
262
|
+
# end while
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
async def fetch_available(self, batch=10, timeout = 0.1):
|
|
266
|
+
"""Fetch only immediately available messages without blocking
|
|
267
|
+
|
|
268
|
+
Path on lack of functionality of JetStreamContext.PullSubscription.fetch
|
|
269
|
+
By default uses only very short timeout to account for network latency
|
|
270
|
+
"""
|
|
271
|
+
import json
|
|
272
|
+
|
|
273
|
+
pull_subscription = self.pull_subscription
|
|
274
|
+
# Get from internal queue first
|
|
275
|
+
msgs = []
|
|
276
|
+
needed = batch
|
|
277
|
+
queue = pull_subscription._sub._pending_queue
|
|
118
278
|
|
|
279
|
+
# First get messages from the internal queue
|
|
280
|
+
while not queue.empty() and needed > 0:
|
|
119
281
|
try:
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
282
|
+
msg = queue.get_nowait()
|
|
283
|
+
pull_subscription._sub._pending_size -= len(msg.data)
|
|
284
|
+
status = JetStreamContext.is_status_msg(msg)
|
|
285
|
+
if not status: # Skip status messages
|
|
286
|
+
msgs.append(msg)
|
|
287
|
+
needed -= 1
|
|
288
|
+
except Exception:
|
|
126
289
|
pass
|
|
127
|
-
except nats.errors.ConnectionClosedError:
|
|
128
|
-
if self.on_connection_close == 'RAISE':
|
|
129
|
-
log.warning(f'Connection closed, raising exception on {self}')
|
|
130
|
-
raise
|
|
131
|
-
elif self.on_connection_close == 'FINISH':
|
|
132
|
-
log.warning(f'Connection closed, finishing iteration on {self}')
|
|
133
|
-
self._stop.set()
|
|
134
|
-
else:
|
|
135
|
-
assert self.on_connection_close == 'WAIT'
|
|
136
|
-
log.warning(f'Connection closed, waiting for reconnect on {self}')
|
|
137
|
-
await asyncio.sleep(1.0)
|
|
138
|
-
n += 1
|
|
139
290
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
# bmsg = None
|
|
173
|
-
# data, meta = {}, {}
|
|
174
|
-
# # First - pull subscription
|
|
175
|
-
# if not self._emptied.is_set():
|
|
176
|
-
# # self.pull_batch = deque()
|
|
177
|
-
# while True:
|
|
178
|
-
# if self._stop.is_set():
|
|
179
|
-
# await self.close()
|
|
180
|
-
# raise MessengerReaderStopped
|
|
181
|
-
# try:
|
|
182
|
-
# if len(self.pull_batch) == 0:
|
|
183
|
-
# log.debug(f"Pulling from {self}")
|
|
184
|
-
# self.pull_batch = deque(await self.pull_subscription.fetch(batch=100, timeout=10.0))
|
|
185
|
-
# log.debug(f"Pulled {len(self.pull_batch)} messages from {self}")
|
|
186
|
-
# bmsg = self.pull_batch.popleft()
|
|
187
|
-
# await bmsg.ack()
|
|
188
|
-
# data, meta = self.messenger.unpack_nats_msg(bmsg)
|
|
189
|
-
# self.messenger.log_msg_trace(data, meta, f"SUB PULL iteration from {self.subject}")
|
|
190
|
-
# meta['receive_mode'] = 'pull'
|
|
191
|
-
# except nats.errors.TimeoutError:
|
|
192
|
-
# if self.push_subscription is None:
|
|
193
|
-
# await self._transform_to_push_sub()
|
|
194
|
-
# # we are creating push subscription but still pulling messages if possible to do not miss any message
|
|
195
|
-
# continue
|
|
196
|
-
# else:
|
|
197
|
-
# # push is ready and pull timeouts, we can switch to push
|
|
198
|
-
# self._emptied.set()
|
|
199
|
-
# except nats.errors.ConnectionClosedError:
|
|
200
|
-
# if self.on_connection_close == 'RAISE':
|
|
201
|
-
# log.warning(f'Connection closed, raising exception on {self}')
|
|
202
|
-
# raise
|
|
203
|
-
# elif self.on_connection_close == 'FINISH':
|
|
204
|
-
# log.warning(f'Connection closed, finishing iteration on {self}')
|
|
205
|
-
# self._stop.set()
|
|
206
|
-
# else:
|
|
207
|
-
# assert self.on_connection_close == 'WAIT'
|
|
208
|
-
# log.warning(f'Connection closed, waiting for reconnect on {self}')
|
|
209
|
-
# await asyncio.sleep(1.0)
|
|
210
|
-
# continue
|
|
211
|
-
# break
|
|
212
|
-
#
|
|
213
|
-
# # Second - push subscription
|
|
214
|
-
# if bmsg is None and self._emptied.is_set():
|
|
215
|
-
# log.debug(f"Waiting for pushed to {self}")
|
|
216
|
-
# while True:
|
|
217
|
-
# if self._stop.is_set():
|
|
218
|
-
# await self.close()
|
|
219
|
-
# raise MessengerReaderStopped
|
|
220
|
-
# try:
|
|
221
|
-
# if self._reconnect_needed.is_set():
|
|
222
|
-
# self._reconnect_needed.clear()
|
|
223
|
-
# raise _ReconnectNeededError
|
|
224
|
-
# bmsg = await self.push_subscription.next_msg()
|
|
225
|
-
# await bmsg.ack()
|
|
226
|
-
# data, meta = self.messenger.unpack_nats_msg(bmsg)
|
|
227
|
-
# self.messenger.log_msg_trace(data, meta, f"SUB PUSH iteration from {self.subject}")
|
|
228
|
-
# meta['receive_mode'] = 'push'
|
|
229
|
-
# if f'{meta["id"]}{meta["ts"]}' in self.id_cache:
|
|
230
|
-
# log.debug(f"Skipping duplicated message from {self.subject}")
|
|
231
|
-
# bmsg = data = meta = None
|
|
232
|
-
# continue
|
|
233
|
-
# break
|
|
234
|
-
# except nats.errors.TimeoutError:
|
|
235
|
-
# pass # keep iterating
|
|
236
|
-
# except (nats.errors.ConnectionClosedError, _ReconnectNeededError):
|
|
237
|
-
# if self.on_connection_close == 'RAISE':
|
|
238
|
-
# log.warning(f'Connection closed, raising exception on {self}')
|
|
239
|
-
# raise
|
|
240
|
-
# elif self.on_connection_close == 'FINISH':
|
|
241
|
-
# log.warning(f'Connection closed, finishing iteration on {self}')
|
|
242
|
-
# self._stop.set()
|
|
243
|
-
# else:
|
|
244
|
-
# assert self.on_connection_close == 'WAIT'
|
|
245
|
-
# log.warning(f'Connection closed, waiting for reconnect on {self}')
|
|
246
|
-
# await asyncio.sleep(1.0)
|
|
247
|
-
# transform_to_pull = True
|
|
248
|
-
# keep_pulling = True
|
|
249
|
-
# break
|
|
250
|
-
#
|
|
251
|
-
# except Exception as e:
|
|
252
|
-
# log.error(f"During interation on {self}, push subscription.next_msg raised {e}")
|
|
253
|
-
# await self.close()
|
|
254
|
-
# raise e
|
|
255
|
-
#
|
|
256
|
-
# if bmsg is None:
|
|
257
|
-
# log.debug(f"No message to return, closing {self}")
|
|
258
|
-
# await self.close()
|
|
259
|
-
# raise MessengerReaderStopped
|
|
260
|
-
# cachedid = f'{meta["id"]}{meta["ts"]}'
|
|
261
|
-
# try:
|
|
262
|
-
# self.id_cache.add(cachedid)
|
|
263
|
-
# except Exception as e:
|
|
264
|
-
# log.error(f"Error adding id: {cachedid} of msg: data={data} meta={meta} to id_cache: {e}")
|
|
265
|
-
# log.debug(f"Returning message {self}")
|
|
266
|
-
# self.last_seq = meta['nats']['seq']
|
|
267
|
-
# self._msg_processed.set()
|
|
268
|
-
# return data, meta
|
|
291
|
+
# If we already have enough messages, return early
|
|
292
|
+
if needed == 0:
|
|
293
|
+
return msgs
|
|
294
|
+
|
|
295
|
+
# Make one no_wait request to get immediately available messages
|
|
296
|
+
next_req = {"batch": needed, "no_wait": True}
|
|
297
|
+
await pull_subscription._nc.publish(
|
|
298
|
+
pull_subscription._nms,
|
|
299
|
+
json.dumps(next_req).encode(),
|
|
300
|
+
pull_subscription._deliver,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
start_time = time.monotonic()
|
|
304
|
+
|
|
305
|
+
# Process the response with a very short timeout
|
|
306
|
+
while needed > 0:
|
|
307
|
+
deadline = timeout - (time.monotonic() - start_time)
|
|
308
|
+
if deadline <= 0:
|
|
309
|
+
break
|
|
310
|
+
|
|
311
|
+
try:
|
|
312
|
+
msg = await pull_subscription._sub.next_msg(timeout=deadline)
|
|
313
|
+
status = JetStreamContext.is_status_msg(msg)
|
|
314
|
+
if not status:
|
|
315
|
+
msgs.append(msg)
|
|
316
|
+
needed -= 1
|
|
317
|
+
elif status == "404": # NO_MESSAGES
|
|
318
|
+
break
|
|
319
|
+
except asyncio.TimeoutError:
|
|
320
|
+
break
|
|
321
|
+
|
|
322
|
+
return msgs
|
|
269
323
|
|
|
270
324
|
async def open(self) -> None:
|
|
271
325
|
if self.pull_subscription is not None:
|
|
@@ -296,9 +350,12 @@ class MsgReader(MsgDriver):
|
|
|
296
350
|
async def _create_pull_subscribtion(self, consumer_conf: ConsumerConfig):
|
|
297
351
|
# Durable consumer is probably not needed (at least problematic)
|
|
298
352
|
# consumer_conf.durable_name = self.name if consumer_conf.durable_name is None else consumer_conf.durable_name
|
|
299
|
-
|
|
353
|
+
ret = await self.connection.js.pull_subscribe(self.subject,
|
|
300
354
|
durable=consumer_conf.durable_name,
|
|
301
355
|
config=consumer_conf)
|
|
356
|
+
ci = await ret.consumer_info()
|
|
357
|
+
return ret
|
|
358
|
+
|
|
302
359
|
async def _reopen(self) -> None:
|
|
303
360
|
try:
|
|
304
361
|
await self.pull_subscription.unsubscribe()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|