voxutils 0.7.3__tar.gz → 0.7.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voxutils-0.7.3 → voxutils-0.7.4}/PKG-INFO +1 -1
- {voxutils-0.7.3 → voxutils-0.7.4}/pyproject.toml +1 -1
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/amqpclient.py +78 -21
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/systemservice.py +58 -30
- {voxutils-0.7.3 → voxutils-0.7.4}/uv.lock +1 -1
- {voxutils-0.7.3 → voxutils-0.7.4}/.gitignore +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/.python-version +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/README.md +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/__init__.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/abc.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/amiclient.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/asyncutils.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/config.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/exceptions.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/httpclient.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/log.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/postgresdriver.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/py.typed +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/retrytimer.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/types.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/src/voxutils/utils.py +0 -0
- {voxutils-0.7.3 → voxutils-0.7.4}/tests/postgresdriver_test.py +0 -0
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from asyncio import (
|
|
3
|
-
AbstractEventLoop,
|
|
4
3
|
CancelledError,
|
|
5
4
|
Event,
|
|
6
5
|
Future,
|
|
6
|
+
Lock,
|
|
7
|
+
Task,
|
|
7
8
|
create_task,
|
|
8
|
-
|
|
9
|
+
get_running_loop,
|
|
9
10
|
sleep,
|
|
10
11
|
wait_for,
|
|
11
12
|
)
|
|
@@ -56,10 +57,12 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
56
57
|
self._status: ClientStatus = ClientStatus.UNINITIALIZED
|
|
57
58
|
self._ready_event: Event = Event()
|
|
58
59
|
self._processing_event: Event = Event()
|
|
60
|
+
self._start_lock: Lock = Lock()
|
|
59
61
|
self._is_shutting_down: bool = False
|
|
60
62
|
self._connection: Connection | None = None
|
|
61
63
|
self._channel_in: Channel | None = None
|
|
62
64
|
self._channel_out: Channel | None = None
|
|
65
|
+
self._consumer_task: Task[None] | None = None
|
|
63
66
|
self._logger_in: StructuredLogger | None = None
|
|
64
67
|
self._loggers_client: dict[str, StructuredLogger] = {}
|
|
65
68
|
if self.logger.isEnabledFor(TRACE):
|
|
@@ -101,8 +104,6 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
101
104
|
return self._ready_event
|
|
102
105
|
|
|
103
106
|
def _reconnect(self, closing: Future[T]) -> None:
|
|
104
|
-
loop: AbstractEventLoop = get_event_loop()
|
|
105
|
-
# loop.run_until_complete(self._disconnect())
|
|
106
107
|
self.logger.info('AMQP connection closed', host=self.config.host)
|
|
107
108
|
if self._is_shutting_down:
|
|
108
109
|
return
|
|
@@ -111,8 +112,8 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
111
112
|
self._channel_out = None
|
|
112
113
|
self._connection = None
|
|
113
114
|
self._ready_event.clear()
|
|
114
|
-
|
|
115
|
-
|
|
115
|
+
# Keep the consumer monitor running; it waits on _ready_event.
|
|
116
|
+
get_running_loop().create_task(self.start())
|
|
116
117
|
|
|
117
118
|
async def _consumer(self, message: DeliveredMessage) -> None:
|
|
118
119
|
await self._processing_event.wait()
|
|
@@ -147,20 +148,47 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
147
148
|
finally:
|
|
148
149
|
self._processing_event.set()
|
|
149
150
|
|
|
151
|
+
async def _ensure_consumer(self) -> None:
|
|
152
|
+
channel: Channel | None = self._channel_in
|
|
153
|
+
if channel is None:
|
|
154
|
+
raise RuntimeError('AMQP input channel is not open')
|
|
155
|
+
queue_name: str = self.config.queue
|
|
156
|
+
if queue_name in channel.consumers:
|
|
157
|
+
return
|
|
158
|
+
try:
|
|
159
|
+
await channel.basic_consume(
|
|
160
|
+
queue_name,
|
|
161
|
+
self._consumer,
|
|
162
|
+
timeout=self.config.timeout,
|
|
163
|
+
consumer_tag=queue_name,
|
|
164
|
+
)
|
|
165
|
+
except BaseException:
|
|
166
|
+
channel.consumers.pop(queue_name, None)
|
|
167
|
+
raise
|
|
168
|
+
|
|
150
169
|
async def _consumer_monitor(self) -> None:
|
|
151
170
|
queue_name: str = self.config.queue
|
|
152
171
|
timeout: int = self.config.timeout
|
|
153
172
|
while True:
|
|
154
173
|
try:
|
|
155
174
|
await self._ready_event.wait()
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
175
|
+
if self._is_shutting_down:
|
|
176
|
+
return
|
|
177
|
+
channel: Channel | None = self._channel_in
|
|
178
|
+
if channel is None or channel.is_closed:
|
|
179
|
+
raise RuntimeError('AMQP input channel is closed')
|
|
180
|
+
# Verify that the channel remains usable. Consumer cancellation
|
|
181
|
+
# notifications remove the callback from channel.consumers.
|
|
182
|
+
await channel.queue_declare(queue_name, durable=True, timeout=timeout)
|
|
183
|
+
if queue_name not in channel.consumers:
|
|
184
|
+
self.logger.warning(
|
|
185
|
+
'AMQP consumer cancelled by broker, re-registering',
|
|
186
|
+
host=self.config.host,
|
|
160
187
|
)
|
|
161
|
-
self.
|
|
188
|
+
await self._ensure_consumer()
|
|
162
189
|
except Exception:
|
|
163
190
|
self.logger.exception('AMQP consumer error', host=self.config.host)
|
|
191
|
+
await self._disconnect()
|
|
164
192
|
if self._is_shutting_down:
|
|
165
193
|
return
|
|
166
194
|
await sleep(5)
|
|
@@ -199,6 +227,10 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
199
227
|
return True
|
|
200
228
|
|
|
201
229
|
async def start(self) -> None:
|
|
230
|
+
async with self._start_lock:
|
|
231
|
+
await self._start()
|
|
232
|
+
|
|
233
|
+
async def _start(self) -> None:
|
|
202
234
|
if self._status in (ClientStatus.CONNECTED, ClientStatus.CONNECTING):
|
|
203
235
|
raise RuntimeError('AMQP client already started')
|
|
204
236
|
|
|
@@ -213,7 +245,8 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
213
245
|
amqp_url: str = f'amqp://{user}:{pswd}@{host}:{port}?heartbeat=1'
|
|
214
246
|
|
|
215
247
|
self._status = ClientStatus.CONNECTING
|
|
216
|
-
|
|
248
|
+
if self._consumer_task is None or self._consumer_task.done():
|
|
249
|
+
self._consumer_task = run_task(self._consumer_monitor(), f'{queue_name} AMQP consumer')
|
|
217
250
|
while True:
|
|
218
251
|
try:
|
|
219
252
|
self.logger.debug('Trying connection to AMQP', host=self.config.host)
|
|
@@ -221,26 +254,40 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
221
254
|
await wait_for(self._connection.connect(), _SOCKET_TIMEOUT)
|
|
222
255
|
self._channel_in = cast(Channel, await self._connection.channel())
|
|
223
256
|
await self._channel_in.basic_qos(prefetch_count=prefetch_count, timeout=timeout)
|
|
224
|
-
await self._channel_in.queue_declare(queue_name, durable=True)
|
|
257
|
+
await self._channel_in.queue_declare(queue_name, durable=True, timeout=timeout)
|
|
258
|
+
await self._ensure_consumer()
|
|
225
259
|
self._channel_out = cast(Channel, await self._connection.channel())
|
|
226
260
|
for client in self.clients:
|
|
227
261
|
await self._channel_out.exchange_declare(
|
|
228
|
-
exchange=client.amqp_exchange,
|
|
262
|
+
exchange=client.amqp_exchange,
|
|
263
|
+
exchange_type='fanout',
|
|
264
|
+
durable=True,
|
|
265
|
+
timeout=timeout,
|
|
229
266
|
)
|
|
230
267
|
arguments: ArgumentsType = {}
|
|
231
268
|
if client.amqp_dead_letter_exchange:
|
|
232
269
|
arguments['x-dead-letter-exchange'] = client.amqp_dead_letter_exchange
|
|
233
270
|
# Queue is assumed to have the same name as the exchange
|
|
234
271
|
await self._channel_out.queue_declare(
|
|
235
|
-
client.amqp_exchange,
|
|
272
|
+
client.amqp_exchange,
|
|
273
|
+
durable=True,
|
|
274
|
+
arguments=arguments,
|
|
275
|
+
timeout=timeout,
|
|
276
|
+
)
|
|
277
|
+
await self._channel_out.queue_bind(
|
|
278
|
+
client.amqp_exchange, client.amqp_exchange, timeout=timeout
|
|
236
279
|
)
|
|
237
|
-
await self._channel_out.queue_bind(client.amqp_exchange, client.amqp_exchange)
|
|
238
280
|
self._connection.closing.add_done_callback(self._reconnect)
|
|
239
281
|
self._status = ClientStatus.CONNECTED
|
|
240
282
|
self.retry_timer.reset()
|
|
241
283
|
self._ready_event.set()
|
|
242
284
|
self.logger.info('AMQP connection established')
|
|
243
285
|
return
|
|
286
|
+
except CancelledError:
|
|
287
|
+
self._status = ClientStatus.ERROR
|
|
288
|
+
if not self._is_shutting_down:
|
|
289
|
+
create_task(self.start())
|
|
290
|
+
raise
|
|
244
291
|
except TimeoutError:
|
|
245
292
|
self.logger.error('AMQP connection attempt timed out', host=self.config.host)
|
|
246
293
|
self._status = ClientStatus.ERROR
|
|
@@ -250,11 +297,9 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
250
297
|
except Exception:
|
|
251
298
|
self.logger.exception('Unexpected AMQP connection error', host=self.config.host)
|
|
252
299
|
self._status = ClientStatus.ERROR
|
|
253
|
-
|
|
254
|
-
self._status
|
|
255
|
-
|
|
256
|
-
create_task(self.start())
|
|
257
|
-
raise
|
|
300
|
+
finally:
|
|
301
|
+
if self._status != ClientStatus.CONNECTED:
|
|
302
|
+
await self._abort_connection()
|
|
258
303
|
|
|
259
304
|
if self._is_shutting_down:
|
|
260
305
|
return
|
|
@@ -277,6 +322,18 @@ class AmqpClient(Generic[_TC, _TP]):
|
|
|
277
322
|
def is_running(self) -> bool:
|
|
278
323
|
return self.status == ClientStatus.CONNECTED
|
|
279
324
|
|
|
325
|
+
async def _abort_connection(self) -> None:
|
|
326
|
+
self._channel_in = None
|
|
327
|
+
self._channel_out = None
|
|
328
|
+
connection: Connection | None = self._connection
|
|
329
|
+
self._connection = None
|
|
330
|
+
if connection is None:
|
|
331
|
+
return
|
|
332
|
+
try:
|
|
333
|
+
await wait_for(connection.close(), _SOCKET_TIMEOUT)
|
|
334
|
+
except (Exception, CancelledError):
|
|
335
|
+
pass
|
|
336
|
+
|
|
280
337
|
async def _disconnect(self) -> None:
|
|
281
338
|
self._ready_event.clear()
|
|
282
339
|
if self._channel_in:
|
|
@@ -683,14 +683,13 @@ class NetService(SystemService[CN], Generic[CN]):
|
|
|
683
683
|
rel_url: str = request.rel_url.human_repr()
|
|
684
684
|
clients: dict[str, ClientConfig] = self._config.clients
|
|
685
685
|
client: str = ''
|
|
686
|
-
client_conf: ClientConfig | None
|
|
686
|
+
client_conf: ClientConfig | None
|
|
687
687
|
response: web.StreamResponse | None
|
|
688
688
|
ip_str: str | None = request.headers.get('X-Real-IP') or request.remote
|
|
689
689
|
self._logger.trace(
|
|
690
690
|
'Received HTTP request', seq=seq, method=request.method, url=rel_url, ip=ip_str
|
|
691
691
|
)
|
|
692
692
|
try:
|
|
693
|
-
# Try to identify client by IP
|
|
694
693
|
try:
|
|
695
694
|
if not ip_str:
|
|
696
695
|
raise ValueError()
|
|
@@ -698,28 +697,31 @@ class NetService(SystemService[CN], Generic[CN]):
|
|
|
698
697
|
except ValueError:
|
|
699
698
|
response = web.Response(status=500, text='Could not detect client IP address')
|
|
700
699
|
return await respond('', response)
|
|
701
|
-
client, client_conf = next(
|
|
702
|
-
(
|
|
703
|
-
(clnt, clnt_conf)
|
|
704
|
-
for clnt, clnt_conf in clients.items()
|
|
705
|
-
for allowed_net in clnt_conf.allowed_ips
|
|
706
|
-
if remote_ip in allowed_net
|
|
707
|
-
),
|
|
708
|
-
('', None),
|
|
709
|
-
)
|
|
710
|
-
|
|
711
700
|
api_key: str = self._get_api_key(request)
|
|
712
|
-
|
|
713
|
-
self._logger.info(
|
|
714
|
-
'Client identified by IP sent wrong API key',
|
|
715
|
-
seq=seq,
|
|
716
|
-
method=request.method,
|
|
717
|
-
url=rel_url,
|
|
718
|
-
ip=ip_str,
|
|
719
|
-
)
|
|
720
|
-
response = web.Response(status=401)
|
|
721
|
-
return await respond(client, response)
|
|
701
|
+
possible_clients: dict[str, ClientConfig] = {}
|
|
722
702
|
if api_key:
|
|
703
|
+
# An API key selects its client directly. Reject it when the source IP
|
|
704
|
+
# already belongs to a different key, and still require the password
|
|
705
|
+
# when that client has one.
|
|
706
|
+
client, client_conf = next(
|
|
707
|
+
(
|
|
708
|
+
(clnt, clnt_conf)
|
|
709
|
+
for clnt, clnt_conf in clients.items()
|
|
710
|
+
for allowed_net in clnt_conf.allowed_ips
|
|
711
|
+
if remote_ip in allowed_net
|
|
712
|
+
),
|
|
713
|
+
('', None),
|
|
714
|
+
)
|
|
715
|
+
if client_conf and client_conf.api_key != api_key:
|
|
716
|
+
self._logger.info(
|
|
717
|
+
'Client identified by IP sent wrong API key',
|
|
718
|
+
seq=seq,
|
|
719
|
+
method=request.method,
|
|
720
|
+
url=rel_url,
|
|
721
|
+
ip=ip_str,
|
|
722
|
+
)
|
|
723
|
+
response = web.Response(status=401)
|
|
724
|
+
return await respond(client, response)
|
|
723
725
|
if request.scheme != 'https':
|
|
724
726
|
if request.headers.get(hdrs.X_FORWARDED_PROTO) != 'https':
|
|
725
727
|
self._logger.info(
|
|
@@ -738,17 +740,43 @@ class NetService(SystemService[CN], Generic[CN]):
|
|
|
738
740
|
),
|
|
739
741
|
('', None),
|
|
740
742
|
)
|
|
741
|
-
|
|
742
|
-
possible_clients: dict[str, ClientConfig] = {}
|
|
743
|
-
if client_conf:
|
|
744
|
-
if client_conf.server_user:
|
|
745
|
-
# Found client by IP and/or API key, but authentication is still required
|
|
743
|
+
if client_conf and client_conf.server_user:
|
|
746
744
|
possible_clients = {client: client_conf}
|
|
747
745
|
client = ''
|
|
746
|
+
client_conf = None
|
|
747
|
+
elif not client_conf:
|
|
748
|
+
possible_clients = {
|
|
749
|
+
clnt: clnt_def for clnt, clnt_def in clients.items() if clnt_def.server_user
|
|
750
|
+
}
|
|
748
751
|
else:
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
+
# A passwordless IP client is used only when this path is allowed for it.
|
|
753
|
+
# Otherwise fall through to basic auth instead of rejecting the request.
|
|
754
|
+
ip_clients: list[tuple[str, ClientConfig]] = [
|
|
755
|
+
(clnt, clnt_conf)
|
|
756
|
+
for clnt, clnt_conf in clients.items()
|
|
757
|
+
if any(remote_ip in allowed_net for allowed_net in clnt_conf.allowed_ips)
|
|
758
|
+
]
|
|
759
|
+
client, client_conf = next(
|
|
760
|
+
(
|
|
761
|
+
(clnt, clnt_conf)
|
|
762
|
+
for clnt, clnt_conf in ip_clients
|
|
763
|
+
if not clnt_conf.server_user
|
|
764
|
+
and not any(pattern.match(rel_url) for pattern in clnt_conf.path_deny)
|
|
765
|
+
and any(pattern.match(rel_url) for pattern in clnt_conf.path_allow)
|
|
766
|
+
),
|
|
767
|
+
('', None),
|
|
768
|
+
)
|
|
769
|
+
if not client_conf:
|
|
770
|
+
password_clients: list[tuple[str, ClientConfig]] = [
|
|
771
|
+
(clnt, clnt_conf) for clnt, clnt_conf in ip_clients if clnt_conf.server_user
|
|
772
|
+
]
|
|
773
|
+
if not password_clients:
|
|
774
|
+
password_clients = [
|
|
775
|
+
(clnt, clnt_conf)
|
|
776
|
+
for clnt, clnt_conf in clients.items()
|
|
777
|
+
if clnt_conf.server_user
|
|
778
|
+
]
|
|
779
|
+
possible_clients = dict(password_clients)
|
|
752
780
|
|
|
753
781
|
if not client_conf:
|
|
754
782
|
self._logger.trace('Performing basic authentication', seq=seq)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|