wherobots-python-dbapi 0.28.2__py3-none-any.whl → 0.29.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wherobots/db/_transport.py +33 -0
- wherobots/db/connection.py +343 -31
- wherobots/db/constants.py +2 -0
- wherobots/db/driver.py +15 -0
- wherobots/db/types.py +3 -0
- {wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/METADATA +6 -1
- {wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/RECORD +9 -9
- {wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/WHEEL +0 -0
- {wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/licenses/LICENSE +0 -0
wherobots/db/_transport.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""Transport termination independent of the WebSocket protocol's send lock."""
|
|
2
2
|
|
|
3
3
|
import errno
|
|
4
|
+
import select
|
|
5
|
+
import selectors
|
|
4
6
|
import socket
|
|
5
7
|
|
|
6
8
|
from websockets.sync.client import ClientConnection
|
|
@@ -22,3 +24,34 @@ def abort_connection(ws: ClientConnection) -> None:
|
|
|
22
24
|
finally:
|
|
23
25
|
# Even if shutdown fails, prevent any later send on this socket object.
|
|
24
26
|
ws.socket.close()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def socket_writable(ws: ClientConnection) -> bool:
|
|
30
|
+
"""Whether a small write to the transport's socket won't block right now.
|
|
31
|
+
|
|
32
|
+
For a send that must not stall the reader thread: a full send buffer
|
|
33
|
+
(a peer that keeps sending but stops reading) would park sendall(), which
|
|
34
|
+
also holds the library's protocol mutex, so even keepalive pings stall.
|
|
35
|
+
|
|
36
|
+
Never raises. Returns ``True`` (proceed, as without this check) on any
|
|
37
|
+
failure or for a transport without a real socket, such as a test fake:
|
|
38
|
+
``False`` would silently disable the caller's sends. Uses ``poll`` (or a
|
|
39
|
+
selector), not ``select.select``, which fails for fds above FD_SETSIZE.
|
|
40
|
+
"""
|
|
41
|
+
try:
|
|
42
|
+
sock = getattr(ws, "socket", None)
|
|
43
|
+
if not isinstance(sock, socket.socket):
|
|
44
|
+
return True
|
|
45
|
+
fd = sock.fileno()
|
|
46
|
+
if fd < 0:
|
|
47
|
+
return True # Closed: let the send itself report it.
|
|
48
|
+
if hasattr(select, "poll"):
|
|
49
|
+
poller = select.poll()
|
|
50
|
+
poller.register(fd, select.POLLOUT)
|
|
51
|
+
# POLLERR/POLLHUP count as writable: the send then fails fast.
|
|
52
|
+
return bool(poller.poll(0))
|
|
53
|
+
with selectors.DefaultSelector() as selector:
|
|
54
|
+
selector.register(fd, selectors.EVENT_WRITE)
|
|
55
|
+
return bool(selector.select(0))
|
|
56
|
+
except Exception:
|
|
57
|
+
return True
|
wherobots/db/connection.py
CHANGED
|
@@ -3,8 +3,9 @@ import logging
|
|
|
3
3
|
import math
|
|
4
4
|
import textwrap
|
|
5
5
|
import threading
|
|
6
|
+
import time
|
|
6
7
|
import uuid
|
|
7
|
-
from dataclasses import dataclass
|
|
8
|
+
from dataclasses import dataclass, field
|
|
8
9
|
from typing import Any, Callable, Dict
|
|
9
10
|
|
|
10
11
|
from wherobots.db.redaction import get_statement_type, redact_sql
|
|
@@ -15,8 +16,8 @@ import cbor2
|
|
|
15
16
|
import websockets.exceptions
|
|
16
17
|
import websockets.sync.client
|
|
17
18
|
|
|
18
|
-
from .constants import DEFAULT_READ_TIMEOUT_SECONDS
|
|
19
|
-
from ._transport import abort_connection
|
|
19
|
+
from .constants import DEFAULT_READ_TIMEOUT_SECONDS, DEFAULT_STALE_QUERY_PROBE_SECONDS
|
|
20
|
+
from ._transport import abort_connection, socket_writable
|
|
20
21
|
from .cursor import Cursor
|
|
21
22
|
from .errors import NotSupportedError, OperationalError
|
|
22
23
|
from .models import ExecutionResult, ProgressInfo, Store, StoreResult
|
|
@@ -69,6 +70,116 @@ def _handshake_bound(ws: Any) -> float | None:
|
|
|
69
70
|
return None
|
|
70
71
|
|
|
71
72
|
|
|
73
|
+
_STALE_CHECK_MAX_INTERVAL_SECONDS = 1.0
|
|
74
|
+
"""Upper bound on how often the reader thread scans for stale queries."""
|
|
75
|
+
|
|
76
|
+
_MAX_PROBE_BACKOFF_DOUBLINGS = 3
|
|
77
|
+
"""Probe spacing doubles from N up to 8N (2**3 * N), then stays there."""
|
|
78
|
+
|
|
79
|
+
_RESULTS_PROBE_AFTER_FACTOR = 4
|
|
80
|
+
"""A query whose results were already requested gets one probe, after 4N.
|
|
81
|
+
|
|
82
|
+
Its terminal event has arrived; only the retrieve_results reply is missing.
|
|
83
|
+
The session re-serializes and re-sends the whole result (up to 100 MiB) for
|
|
84
|
+
every retrieve, synchronously on its event loop, so duplicates are costly. A
|
|
85
|
+
lost reply is also the rarer case: retrieve replies are sent inline on the
|
|
86
|
+
session's main loop, not pushed from a callback like state_updated.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
_EXECUTION_NOT_FOUND = "Execution not found"
|
|
90
|
+
"""sql-session's error for a retrieve_results it has no cache entry for.
|
|
91
|
+
|
|
92
|
+
Its execution cache is a small LRU that can evict a query that is still
|
|
93
|
+
running, whose terminal state_updated still arrives (it's pushed from the
|
|
94
|
+
query's own done-callback). So for a probe, this means "can't answer", not
|
|
95
|
+
"failed".
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
_PROBED_STATES = frozenset(
|
|
99
|
+
{
|
|
100
|
+
ExecutionState.EXECUTION_REQUESTED,
|
|
101
|
+
ExecutionState.PENDING,
|
|
102
|
+
ExecutionState.RUNNING,
|
|
103
|
+
ExecutionState.RESULTS_REQUESTED,
|
|
104
|
+
}
|
|
105
|
+
)
|
|
106
|
+
"""States in which the driver is waiting on a session event it can re-ask for."""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _probe_interval(value: Any) -> float | None:
|
|
110
|
+
"""The validated stale-query probe interval, or ``None`` to disable it.
|
|
111
|
+
|
|
112
|
+
Like ``_handshake_bound``, returns ``None`` for a non-numeric (including
|
|
113
|
+
``bool``), non-finite or non-positive value. Never raises ``Exception``:
|
|
114
|
+
``connect()`` callers pass this through, and a bad value must not cost
|
|
115
|
+
them their connection.
|
|
116
|
+
"""
|
|
117
|
+
try:
|
|
118
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
119
|
+
return None
|
|
120
|
+
value = float(value)
|
|
121
|
+
if not math.isfinite(value) or value <= 0:
|
|
122
|
+
return None
|
|
123
|
+
return value
|
|
124
|
+
except Exception:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
_INVALID_VALUE_REPR_CHARS = 80
|
|
129
|
+
"""Longest repr of an invalid setting that a warning quotes."""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _describe_invalid(value: Any) -> str:
|
|
133
|
+
"""A bounded ``repr`` of ``value`` for a warning. Never raises ``Exception``.
|
|
134
|
+
|
|
135
|
+
Truncated so a huge value (``10**400``) can't flood the log; falls back to
|
|
136
|
+
the type name if ``repr`` fails, since this runs in the constructor.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
text = repr(value)
|
|
140
|
+
except Exception:
|
|
141
|
+
try:
|
|
142
|
+
return f"<unrepresentable {type(value).__name__}>"
|
|
143
|
+
except Exception:
|
|
144
|
+
return "<unrepresentable>"
|
|
145
|
+
if len(text) > _INVALID_VALUE_REPR_CHARS:
|
|
146
|
+
text = text[: _INVALID_VALUE_REPR_CHARS - 3] + "..."
|
|
147
|
+
return text
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass
|
|
151
|
+
class _Watch:
|
|
152
|
+
"""Stale-query watchdog bookkeeping for one query.
|
|
153
|
+
|
|
154
|
+
Only the reader thread touches it once the query is registered; ``__send``
|
|
155
|
+
stamps ``last_activity`` and ``last_event`` under ``__lock`` just before
|
|
156
|
+
registration.
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
last_activity: float = field(default_factory=time.monotonic)
|
|
160
|
+
"""``time.monotonic()`` of the last inbound event (or probe) for the query.
|
|
161
|
+
|
|
162
|
+
Spaces the probes; see ``last_event`` for the true silence."""
|
|
163
|
+
last_event: float = field(default_factory=time.monotonic)
|
|
164
|
+
"""``time.monotonic()`` of the last inbound event (or registration).
|
|
165
|
+
|
|
166
|
+
Unlike ``last_activity``, probes don't update it, so the probe log reports
|
|
167
|
+
how long the query has really been silent."""
|
|
168
|
+
probes_sent: int = 0
|
|
169
|
+
"""Probes sent since the last event that wasn't a reply to a retrieve."""
|
|
170
|
+
probe_outstanding: bool = False
|
|
171
|
+
"""A probe was sent and no reply to a retrieve has arrived since.
|
|
172
|
+
|
|
173
|
+
Other events, a non-terminal ``state_updated`` included, leave it set: they
|
|
174
|
+
don't answer the probe, whose reply may still be on its way."""
|
|
175
|
+
probeable: bool = True
|
|
176
|
+
"""False once the session has said it can't answer probes for the query."""
|
|
177
|
+
results_requested: bool = False
|
|
178
|
+
"""The normal (non-probe) retrieve_results was sent."""
|
|
179
|
+
results_probed: bool = False
|
|
180
|
+
"""The one probe allowed after ``results_requested`` was sent."""
|
|
181
|
+
|
|
182
|
+
|
|
72
183
|
@dataclass
|
|
73
184
|
class Query:
|
|
74
185
|
sql: str
|
|
@@ -76,6 +187,7 @@ class Query:
|
|
|
76
187
|
state: ExecutionState
|
|
77
188
|
handler: Callable[[Any], None]
|
|
78
189
|
store: Store | None = None
|
|
190
|
+
watch: _Watch = field(default_factory=_Watch)
|
|
79
191
|
|
|
80
192
|
|
|
81
193
|
class Connection:
|
|
@@ -101,13 +213,29 @@ class Connection:
|
|
|
101
213
|
data_compression: DataCompression | None = None,
|
|
102
214
|
geometry_representation: GeometryRepresentation | None = None,
|
|
103
215
|
session_id: str | None = None,
|
|
216
|
+
stale_query_probe_seconds: float | None = DEFAULT_STALE_QUERY_PROBE_SECONDS,
|
|
104
217
|
):
|
|
105
218
|
self.__ws = ws
|
|
106
|
-
self.__read_timeout = read_timeout
|
|
107
219
|
self.__results_format = results_format
|
|
108
220
|
self.__data_compression = data_compression
|
|
109
221
|
self.__geometry_representation = geometry_representation
|
|
110
222
|
self.__progress_handler: ProgressHandler | None = None
|
|
223
|
+
self.__probe_after = _probe_interval(stale_query_probe_seconds)
|
|
224
|
+
if self.__probe_after is None and stale_query_probe_seconds is not None:
|
|
225
|
+
logging.warning(
|
|
226
|
+
"Invalid stale_query_probe_seconds (%s); stale-query probes disabled.",
|
|
227
|
+
_describe_invalid(stale_query_probe_seconds),
|
|
228
|
+
)
|
|
229
|
+
self.__next_stale_check = 0.0
|
|
230
|
+
# recv() must wake up for the watchdog's checks, also when the caller
|
|
231
|
+
# asked for no read timeout (websockets accepts None) or a long one.
|
|
232
|
+
self.__recv_timeout = read_timeout
|
|
233
|
+
if self.__probe_after is not None:
|
|
234
|
+
check = min(_STALE_CHECK_MAX_INTERVAL_SECONDS, self.__probe_after)
|
|
235
|
+
if read_timeout is None or (
|
|
236
|
+
isinstance(read_timeout, (int, float)) and read_timeout > check
|
|
237
|
+
):
|
|
238
|
+
self.__recv_timeout = check
|
|
111
239
|
|
|
112
240
|
self.__session_id = session_id
|
|
113
241
|
self.__lock = threading.Lock()
|
|
@@ -200,6 +328,10 @@ class Connection:
|
|
|
200
328
|
while True:
|
|
201
329
|
if self.__shutdown_done.is_set():
|
|
202
330
|
return
|
|
331
|
+
# Checked on every iteration, not only when recv() times out: on a
|
|
332
|
+
# busy connection, steady traffic for one query would otherwise
|
|
333
|
+
# starve another query's watchdog.
|
|
334
|
+
self.__probe_stale_queries()
|
|
203
335
|
try:
|
|
204
336
|
self.__listen()
|
|
205
337
|
except TimeoutError:
|
|
@@ -214,6 +346,73 @@ class Connection:
|
|
|
214
346
|
except Exception as e:
|
|
215
347
|
logging.exception("Error handling message from SQL session", exc_info=e)
|
|
216
348
|
|
|
349
|
+
def __probe_stale_queries(self) -> None:
|
|
350
|
+
"""Re-ask the session about queries that have gone quiet.
|
|
351
|
+
|
|
352
|
+
Defence in depth for a lost terminal event: the driver otherwise trusts
|
|
353
|
+
a single pushed ``state_updated`` and would wait forever if it never
|
|
354
|
+
arrives. A probe is a ``retrieve_results`` request; the session answers
|
|
355
|
+
with the query's current state (and its results or result location
|
|
356
|
+
once it has succeeded), or with an error for an unknown execution.
|
|
357
|
+
|
|
358
|
+
Runs on the reader thread, which owns every ``Query``'s watchdog fields.
|
|
359
|
+
Never raises: a watchdog failure must not disturb the receive loop.
|
|
360
|
+
"""
|
|
361
|
+
probe_after = self.__probe_after
|
|
362
|
+
if probe_after is None:
|
|
363
|
+
return
|
|
364
|
+
try:
|
|
365
|
+
now = time.monotonic()
|
|
366
|
+
if now < self.__next_stale_check:
|
|
367
|
+
return
|
|
368
|
+
self.__next_stale_check = now + min(
|
|
369
|
+
_STALE_CHECK_MAX_INTERVAL_SECONDS, probe_after
|
|
370
|
+
)
|
|
371
|
+
with self.__lock:
|
|
372
|
+
if self.__closed:
|
|
373
|
+
return
|
|
374
|
+
candidates = list(self.__queries.values())
|
|
375
|
+
for query in candidates:
|
|
376
|
+
watch = query.watch
|
|
377
|
+
if not watch.probeable or query.state not in _PROBED_STATES:
|
|
378
|
+
continue
|
|
379
|
+
if watch.results_requested:
|
|
380
|
+
if watch.results_probed:
|
|
381
|
+
continue
|
|
382
|
+
wait = probe_after * _RESULTS_PROBE_AFTER_FACTOR
|
|
383
|
+
else:
|
|
384
|
+
doublings = min(watch.probes_sent, _MAX_PROBE_BACKOFF_DOUBLINGS)
|
|
385
|
+
wait = probe_after * 2**doublings
|
|
386
|
+
silence = now - watch.last_activity
|
|
387
|
+
if silence < wait:
|
|
388
|
+
continue
|
|
389
|
+
if not self.__request_results(query.execution_id, probe=True):
|
|
390
|
+
# A user thread is sending, possibly stalled, or the
|
|
391
|
+
# socket can't take a write. Don't wait: that would stop
|
|
392
|
+
# recv() for every query. Either holds for the rest too;
|
|
393
|
+
# retry at the next check.
|
|
394
|
+
break
|
|
395
|
+
# Only a reply to a retrieve clears the flag; other events,
|
|
396
|
+
# state_updated included, don't. Still set means the last
|
|
397
|
+
# probe went unanswered.
|
|
398
|
+
unanswered = watch.probe_outstanding
|
|
399
|
+
watch.probes_sent += 1
|
|
400
|
+
watch.probe_outstanding = True
|
|
401
|
+
watch.results_probed = watch.results_requested
|
|
402
|
+
# Space the next probe from this one, not from the last event.
|
|
403
|
+
watch.last_activity = now
|
|
404
|
+
logging.info(
|
|
405
|
+
"No events for query %s in %.1fs while %s; probed the SQL "
|
|
406
|
+
"session for its state (probe %d%s).",
|
|
407
|
+
query.execution_id,
|
|
408
|
+
now - watch.last_event,
|
|
409
|
+
query.state,
|
|
410
|
+
watch.probes_sent,
|
|
411
|
+
", previous probe unanswered" if unanswered else "",
|
|
412
|
+
)
|
|
413
|
+
except Exception:
|
|
414
|
+
logging.exception("Stale-query probe failed")
|
|
415
|
+
|
|
217
416
|
def __connection_error(self, execution_id: str) -> OperationalError:
|
|
218
417
|
message = (
|
|
219
418
|
f"SQL connection lost (session={self.__session_id or 'unknown'}, "
|
|
@@ -299,6 +498,27 @@ class Connection:
|
|
|
299
498
|
# Invalid event.
|
|
300
499
|
return
|
|
301
500
|
|
|
501
|
+
query = self.__queries.get(execution_id)
|
|
502
|
+
# Whether this event answers a probe (possibly along with the normal
|
|
503
|
+
# retrieve). Read before the bookkeeping below resets it.
|
|
504
|
+
probe_reply = False
|
|
505
|
+
if query is not None:
|
|
506
|
+
watch = query.watch
|
|
507
|
+
# Any inbound event, progress included, shows the query is alive.
|
|
508
|
+
watch.last_activity = watch.last_event = time.monotonic()
|
|
509
|
+
if kind == EventKind.EXECUTION_RESULT or kind == EventKind.ERROR:
|
|
510
|
+
# A reply to a retrieve_results, which may be our own probe: it
|
|
511
|
+
# resets the timer but not the backoff, so a long-running
|
|
512
|
+
# silent query isn't probed every N seconds forever.
|
|
513
|
+
probe_reply = watch.probe_outstanding
|
|
514
|
+
watch.probe_outstanding = False
|
|
515
|
+
else:
|
|
516
|
+
# A genuine event resets the backoff, but leaves an outstanding
|
|
517
|
+
# probe outstanding: a non-terminal state_updated can arrive
|
|
518
|
+
# between a probe and its reply, which must still be
|
|
519
|
+
# recognised as the probe's (see the not-found guard below).
|
|
520
|
+
watch.probes_sent = 0
|
|
521
|
+
|
|
302
522
|
# Progress events are independent of the query state machine and don't
|
|
303
523
|
# require a tracked query — the handler is connection-level.
|
|
304
524
|
if kind == EventKind.EXECUTION_PROGRESS:
|
|
@@ -318,7 +538,6 @@ class Connection:
|
|
|
318
538
|
logging.exception("Progress handler raised an exception")
|
|
319
539
|
return
|
|
320
540
|
|
|
321
|
-
query = self.__queries.get(execution_id)
|
|
322
541
|
if not query:
|
|
323
542
|
logging.warning(
|
|
324
543
|
"Received %s event for unknown execution ID %s", kind, execution_id
|
|
@@ -332,6 +551,18 @@ class Connection:
|
|
|
332
551
|
with self.__lock:
|
|
333
552
|
claimed = self.__queries.pop(execution_id, None)
|
|
334
553
|
if claimed is not None:
|
|
554
|
+
if probe_reply and result.error is None:
|
|
555
|
+
if claimed.watch.results_requested:
|
|
556
|
+
# The normal retrieve's reply may have won the race.
|
|
557
|
+
logging.info(
|
|
558
|
+
"Query %s results arrived after a stale-query probe.",
|
|
559
|
+
execution_id,
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
logging.info(
|
|
563
|
+
"Query %s recovered from a stale-query probe reply.",
|
|
564
|
+
execution_id,
|
|
565
|
+
)
|
|
335
566
|
claimed.handler(result)
|
|
336
567
|
|
|
337
568
|
def fail_query(action: str, error: Exception) -> None:
|
|
@@ -348,6 +579,18 @@ class Connection:
|
|
|
348
579
|
logging.error("%s", message)
|
|
349
580
|
complete_query(ExecutionResult(error=OperationalError(message)))
|
|
350
581
|
|
|
582
|
+
def complete_with_store_result(result_uri: str) -> None:
|
|
583
|
+
# Results are stored in cloud storage
|
|
584
|
+
store_result = StoreResult(result_uri=result_uri, size=message.get("size"))
|
|
585
|
+
logging.info(
|
|
586
|
+
"Query %s results stored at: %s (size: %s)",
|
|
587
|
+
execution_id,
|
|
588
|
+
result_uri,
|
|
589
|
+
store_result.size,
|
|
590
|
+
)
|
|
591
|
+
query.state = ExecutionState.COMPLETED
|
|
592
|
+
complete_query(ExecutionResult(store_result=store_result))
|
|
593
|
+
|
|
351
594
|
# Incoming state transitions are handled here.
|
|
352
595
|
if kind == EventKind.STATE_UPDATED or kind == EventKind.EXECUTION_RESULT:
|
|
353
596
|
try:
|
|
@@ -363,19 +606,7 @@ class Connection:
|
|
|
363
606
|
if kind == EventKind.STATE_UPDATED:
|
|
364
607
|
result_uri = message.get("result_uri")
|
|
365
608
|
if result_uri:
|
|
366
|
-
|
|
367
|
-
store_result = StoreResult(
|
|
368
|
-
result_uri=result_uri,
|
|
369
|
-
size=message.get("size"),
|
|
370
|
-
)
|
|
371
|
-
logging.info(
|
|
372
|
-
"Query %s results stored at: %s (size: %s)",
|
|
373
|
-
execution_id,
|
|
374
|
-
result_uri,
|
|
375
|
-
store_result.size,
|
|
376
|
-
)
|
|
377
|
-
query.state = ExecutionState.COMPLETED
|
|
378
|
-
complete_query(ExecutionResult(store_result=store_result))
|
|
609
|
+
complete_with_store_result(result_uri)
|
|
379
610
|
return
|
|
380
611
|
|
|
381
612
|
if query.store is not None:
|
|
@@ -397,8 +628,34 @@ class Connection:
|
|
|
397
628
|
fail_query("request", error)
|
|
398
629
|
return
|
|
399
630
|
|
|
400
|
-
# Otherwise,
|
|
631
|
+
# Otherwise, this execution_result answers a retrieve_results.
|
|
632
|
+
# For a store-configured query, that can only be a stale-query
|
|
633
|
+
# probe: the normal path never requests results for one.
|
|
634
|
+
# Sessions with the probe contract echo the store location.
|
|
635
|
+
result_uri = message.get("result_uri")
|
|
636
|
+
if result_uri:
|
|
637
|
+
complete_with_store_result(result_uri)
|
|
638
|
+
return
|
|
639
|
+
|
|
401
640
|
results = message.get("results")
|
|
641
|
+
if query.store is not None and (results is None or results == {}):
|
|
642
|
+
# Succeeded without a result location: either an older
|
|
643
|
+
# session that doesn't echo it, or an empty store result
|
|
644
|
+
# (no store_path). The two are indistinguishable, and
|
|
645
|
+
# completing empty would silently drop an old session's
|
|
646
|
+
# real result. Keep waiting for the genuine state_updated,
|
|
647
|
+
# exactly as without the watchdog, and don't probe again:
|
|
648
|
+
# the answer won't change. Known limitation: an empty store
|
|
649
|
+
# result whose terminal event was lost can't be recovered.
|
|
650
|
+
query.watch.probeable = False
|
|
651
|
+
logging.warning(
|
|
652
|
+
"Query %s succeeded, but the SQL session's reply has no "
|
|
653
|
+
"result location (older SQL session, or an empty store "
|
|
654
|
+
"result); cannot recover it, still waiting.",
|
|
655
|
+
execution_id,
|
|
656
|
+
)
|
|
657
|
+
return
|
|
658
|
+
|
|
402
659
|
if results is None or results == {}:
|
|
403
660
|
logging.warning("Got no results back from %s.", execution_id)
|
|
404
661
|
query.state = ExecutionState.COMPLETED
|
|
@@ -428,8 +685,28 @@ class Connection:
|
|
|
428
685
|
# details.
|
|
429
686
|
pass
|
|
430
687
|
elif kind == EventKind.ERROR:
|
|
431
|
-
query.state = ExecutionState.FAILED
|
|
432
688
|
error = message.get("message")
|
|
689
|
+
if (
|
|
690
|
+
probe_reply
|
|
691
|
+
and error == _EXECUTION_NOT_FOUND
|
|
692
|
+
and not query.watch.results_requested
|
|
693
|
+
and query.state != ExecutionState.FAILED
|
|
694
|
+
):
|
|
695
|
+
# The session evicted the query from its cache. If the query is
|
|
696
|
+
# still running, its terminal event will still arrive; if that
|
|
697
|
+
# event was already lost, nothing more can be done (still
|
|
698
|
+
# running and finished-with-a-lost-event look the same from
|
|
699
|
+
# here), so keep waiting either way. After the normal
|
|
700
|
+
# retrieve, though, there's nothing left to wait for.
|
|
701
|
+
query.watch.probeable = False
|
|
702
|
+
logging.warning(
|
|
703
|
+
"SQL session no longer tracks execution %s (likely evicted "
|
|
704
|
+
"from its result cache); cannot probe it, waiting for its "
|
|
705
|
+
"terminal event.",
|
|
706
|
+
execution_id,
|
|
707
|
+
)
|
|
708
|
+
return
|
|
709
|
+
query.state = ExecutionState.FAILED
|
|
433
710
|
complete_query(ExecutionResult(error=OperationalError(error)))
|
|
434
711
|
else:
|
|
435
712
|
logging.warning("Received unknown %s event!", kind)
|
|
@@ -456,7 +733,17 @@ class Connection:
|
|
|
456
733
|
else:
|
|
457
734
|
raise NotSupportedError("Unsupported results format")
|
|
458
735
|
|
|
459
|
-
def __send(
|
|
736
|
+
def __send(
|
|
737
|
+
self,
|
|
738
|
+
message: Dict[str, Any],
|
|
739
|
+
query: Query | None = None,
|
|
740
|
+
blocking: bool = True,
|
|
741
|
+
) -> bool:
|
|
742
|
+
"""Send a request; return ``False`` only if non-blocking and busy.
|
|
743
|
+
|
|
744
|
+
Busy means another sender holds the send lock, or the socket's send
|
|
745
|
+
buffer is full.
|
|
746
|
+
"""
|
|
460
747
|
# Serialization and redaction are local work. Fail before registration,
|
|
461
748
|
# without poisoning unrelated cursors or misreporting a transport loss.
|
|
462
749
|
request = json.dumps(message)
|
|
@@ -465,16 +752,30 @@ class Connection:
|
|
|
465
752
|
# unguarded call would redact on every request even with DEBUG off.
|
|
466
753
|
if logging.getLogger().isEnabledFor(logging.DEBUG):
|
|
467
754
|
logging.debug("Request: %s", self.__redacted_request(message))
|
|
468
|
-
|
|
755
|
+
if not self.__send_lock.acquire(blocking=blocking):
|
|
756
|
+
return False
|
|
757
|
+
try:
|
|
469
758
|
with self.__lock:
|
|
470
759
|
if self.__closed:
|
|
471
760
|
if query is not None:
|
|
472
761
|
raise self.__connection_error(query.execution_id)
|
|
473
|
-
return
|
|
762
|
+
return True
|
|
474
763
|
if query is not None:
|
|
764
|
+
# The watchdog's clock starts at registration.
|
|
765
|
+
watch = query.watch
|
|
766
|
+
watch.last_activity = watch.last_event = time.monotonic()
|
|
475
767
|
self.__queries[query.execution_id] = query
|
|
476
768
|
elif message.get("execution_id") not in self.__queries:
|
|
477
|
-
return
|
|
769
|
+
return True
|
|
770
|
+
# A non-blocking send runs on the reader thread (a probe). Not
|
|
771
|
+
# waiting for the lock isn't enough: with the socket's send buffer
|
|
772
|
+
# full (a peer that keeps sending but stops reading), ws.send()
|
|
773
|
+
# itself would block, stopping recv() for every query, and would
|
|
774
|
+
# hold the library's protocol mutex, which keepalive pings need
|
|
775
|
+
# too. Treat it like a busy lock: skip, and retry at the next
|
|
776
|
+
# check. Checked outside __lock: it's a syscall on the transport.
|
|
777
|
+
if not blocking and not socket_writable(self.__ws):
|
|
778
|
+
return False
|
|
478
779
|
try:
|
|
479
780
|
self.__ws.send(request)
|
|
480
781
|
except (websockets.exceptions.ConnectionClosed, OSError):
|
|
@@ -486,8 +787,11 @@ class Connection:
|
|
|
486
787
|
self.__queries.pop(query.execution_id, None)
|
|
487
788
|
raise
|
|
488
789
|
else:
|
|
489
|
-
return
|
|
790
|
+
return True
|
|
791
|
+
finally:
|
|
792
|
+
self.__send_lock.release()
|
|
490
793
|
self.__fail_pending()
|
|
794
|
+
return True
|
|
491
795
|
|
|
492
796
|
@staticmethod
|
|
493
797
|
def __redacted_request(message: Dict[str, Any]) -> str:
|
|
@@ -504,7 +808,7 @@ class Connection:
|
|
|
504
808
|
|
|
505
809
|
def __recv(self) -> Dict[str, Any]:
|
|
506
810
|
try:
|
|
507
|
-
frame = self.__ws.recv(timeout=self.
|
|
811
|
+
frame = self.__ws.recv(timeout=self.__recv_timeout)
|
|
508
812
|
except TimeoutError:
|
|
509
813
|
raise # Idle polls are expected, not terminal I/O failures.
|
|
510
814
|
except OSError as e:
|
|
@@ -559,10 +863,15 @@ class Connection:
|
|
|
559
863
|
)
|
|
560
864
|
return execution_id
|
|
561
865
|
|
|
562
|
-
def __request_results(self, execution_id: str) ->
|
|
866
|
+
def __request_results(self, execution_id: str, probe: bool = False) -> bool:
|
|
867
|
+
"""Send retrieve_results; ``False`` if a probe found the sender busy.
|
|
868
|
+
|
|
869
|
+
A probe never waits for ``__send_lock`` or a full socket (see
|
|
870
|
+
``__probe_stale_queries`` and ``__send``).
|
|
871
|
+
"""
|
|
563
872
|
query = self.__queries.get(execution_id)
|
|
564
873
|
if not query:
|
|
565
|
-
return
|
|
874
|
+
return True
|
|
566
875
|
|
|
567
876
|
request = {
|
|
568
877
|
"kind": RequestKind.RETRIEVE_RESULTS.value,
|
|
@@ -575,9 +884,12 @@ class Connection:
|
|
|
575
884
|
if self.__geometry_representation:
|
|
576
885
|
request["geometry"] = self.__geometry_representation.value
|
|
577
886
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
887
|
+
if not probe:
|
|
888
|
+
# A probe only asks; the reply carries the query's actual state.
|
|
889
|
+
query.state = ExecutionState.RESULTS_REQUESTED
|
|
890
|
+
query.watch.results_requested = True
|
|
891
|
+
logging.info("Requesting results from %s ...", execution_id)
|
|
892
|
+
return self.__send(request, blocking=not probe)
|
|
581
893
|
|
|
582
894
|
def __cancel_query(self, execution_id: str) -> None:
|
|
583
895
|
"""Cancels the query with the given execution ID."""
|
wherobots/db/constants.py
CHANGED
|
@@ -17,6 +17,8 @@ DEFAULT_READ_TIMEOUT_SECONDS: float = 0.25
|
|
|
17
17
|
# Bound on the WebSocket close handshake performed by a graceful close().
|
|
18
18
|
DEFAULT_CLOSE_TIMEOUT_SECONDS: float = 1.0
|
|
19
19
|
DEFAULT_SESSION_WAIT_TIMEOUT_SECONDS: float = 900
|
|
20
|
+
# Silence, per query, before the driver re-asks the session for its state.
|
|
21
|
+
DEFAULT_STALE_QUERY_PROBE_SECONDS: float = 30.0
|
|
20
22
|
|
|
21
23
|
MAX_MESSAGE_SIZE: int = 100 * 2**20 # 100MiB
|
|
22
24
|
PROTOCOL_VERSION: Version = Version("1.0.0")
|
wherobots/db/driver.py
CHANGED
|
@@ -25,6 +25,7 @@ from .constants import (
|
|
|
25
25
|
DEFAULT_READ_TIMEOUT_SECONDS,
|
|
26
26
|
DEFAULT_SESSION_TYPE,
|
|
27
27
|
DEFAULT_SESSION_WAIT_TIMEOUT_SECONDS,
|
|
28
|
+
DEFAULT_STALE_QUERY_PROBE_SECONDS,
|
|
28
29
|
MAX_MESSAGE_SIZE,
|
|
29
30
|
PARAM_STYLE,
|
|
30
31
|
PROTOCOL_VERSION,
|
|
@@ -125,6 +126,7 @@ def connect(
|
|
|
125
126
|
geometry_representation: Union[GeometryRepresentation, None] = None,
|
|
126
127
|
cancel_event: Union[threading.Event, None] = None,
|
|
127
128
|
extra_headers: Union[Dict[str, str], None] = None,
|
|
129
|
+
stale_query_probe_seconds: Union[float, None] = DEFAULT_STALE_QUERY_PROBE_SECONDS,
|
|
128
130
|
) -> Connection:
|
|
129
131
|
"""Create a connection to a Wherobots SQL session.
|
|
130
132
|
|
|
@@ -146,6 +148,12 @@ def connect(
|
|
|
146
148
|
own ``client=dbapi;ver=<version>`` hop to the right of it. This header
|
|
147
149
|
is advisory only and never affects authentication; ``extra_headers``
|
|
148
150
|
cannot be used to override the ``Authorization``/``X-API-Key`` headers.
|
|
151
|
+
:param stale_query_probe_seconds: How long a running query may go without
|
|
152
|
+
any event from the SQL session before the driver re-asks the session
|
|
153
|
+
for its state, in case its completion event was lost. Later probes
|
|
154
|
+
back off to at most 8 times this interval. ``None`` disables probing;
|
|
155
|
+
an invalid value (non-numeric, non-finite, or not positive) also
|
|
156
|
+
disables it.
|
|
149
157
|
"""
|
|
150
158
|
if not token and not api_key:
|
|
151
159
|
raise ValueError("At least one of `token` or `api_key` is required")
|
|
@@ -271,6 +279,7 @@ def connect(
|
|
|
271
279
|
session_id=urllib.parse.urlparse(session_id_url)
|
|
272
280
|
.path.rstrip("/")
|
|
273
281
|
.rsplit("/", 1)[-1],
|
|
282
|
+
stale_query_probe_seconds=stale_query_probe_seconds,
|
|
274
283
|
)
|
|
275
284
|
|
|
276
285
|
|
|
@@ -299,7 +308,12 @@ def connect_direct(
|
|
|
299
308
|
geometry_representation: Union[GeometryRepresentation, None] = None,
|
|
300
309
|
cancel_event: Union[threading.Event, None] = None,
|
|
301
310
|
session_id: str | None = None,
|
|
311
|
+
stale_query_probe_seconds: Union[float, None] = DEFAULT_STALE_QUERY_PROBE_SECONDS,
|
|
302
312
|
) -> Connection:
|
|
313
|
+
"""Connect to a SQL session's WebSocket endpoint directly.
|
|
314
|
+
|
|
315
|
+
:param stale_query_probe_seconds: See :func:`connect`.
|
|
316
|
+
"""
|
|
303
317
|
uri_with_protocol = f"{uri}/{protocol}"
|
|
304
318
|
ssl_context = ssl.create_default_context()
|
|
305
319
|
ssl_context.load_verify_locations(certifi.where())
|
|
@@ -344,4 +358,5 @@ def connect_direct(
|
|
|
344
358
|
data_compression=data_compression,
|
|
345
359
|
geometry_representation=geometry_representation,
|
|
346
360
|
session_id=session_id,
|
|
361
|
+
stale_query_probe_seconds=stale_query_probe_seconds,
|
|
347
362
|
)
|
wherobots/db/types.py
CHANGED
|
@@ -9,6 +9,9 @@ class ExecutionState(LowercaseStrEnum):
|
|
|
9
9
|
EXECUTION_REQUESTED = auto()
|
|
10
10
|
"Execution of a query has been requested by the driver."
|
|
11
11
|
|
|
12
|
+
PENDING = auto()
|
|
13
|
+
"The SQL session has accepted the query but not started running it yet."
|
|
14
|
+
|
|
12
15
|
RUNNING = auto()
|
|
13
16
|
"The SQL session has reported the query is running."
|
|
14
17
|
|
{wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: wherobots-python-dbapi
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.29.0
|
|
4
4
|
Summary: Python DB-API driver for Wherobots DB
|
|
5
5
|
Project-URL: Homepage, https://github.com/wherobots/wherobots-python-dbapi-driver
|
|
6
6
|
Project-URL: Tracker, https://github.com/wherobots/wherobots-python-dbapi-driver/issues
|
|
@@ -302,3 +302,8 @@ users may find useful:
|
|
|
302
302
|
caller needs to interrupt it (e.g. on client disconnect or timeout).
|
|
303
303
|
The event is checked before each HTTP request, between retry
|
|
304
304
|
attempts, and before the WebSocket handshake.
|
|
305
|
+
* `stale_query_probe_seconds`: how long a running query may go without
|
|
306
|
+
any message from the SQL session before the driver asks the session
|
|
307
|
+
for its state again, in case the query's completion event was lost.
|
|
308
|
+
Repeat probes back off to at most 8 times this interval. Defaults to
|
|
309
|
+
30 seconds; pass `None` to disable probing.
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
wherobots/db/__init__.py,sha256=HN_uGOUMor80GB3rZIqnPcYWDAia7QAC-y9HfGKnegs,744
|
|
2
|
-
wherobots/db/_transport.py,sha256=
|
|
3
|
-
wherobots/db/connection.py,sha256=
|
|
4
|
-
wherobots/db/constants.py,sha256=
|
|
2
|
+
wherobots/db/_transport.py,sha256=eNe8YN_MqOTga3XNllCLFG69GQbzwEjMXBJsBLM3BEI,2256
|
|
3
|
+
wherobots/db/connection.py,sha256=0_ccXOL9Yq2yalZBbWOoeBaTkympZK30dDs4DWAXHjE,39395
|
|
4
|
+
wherobots/db/constants.py,sha256=m360b8GZ7mVgUcgLUgGu76gwQc1pOcEg7qTRTy9W3eg,984
|
|
5
5
|
wherobots/db/cursor.py,sha256=jF9v4UhfBqZ0iZEAmIYRxWYU4E8UbMFIrwfY_PlNI60,8701
|
|
6
|
-
wherobots/db/driver.py,sha256=
|
|
6
|
+
wherobots/db/driver.py,sha256=1vlwf4RtjlCBeONqkU82zpwcZVT5FAsoL5Vuoxz_xmE,14113
|
|
7
7
|
wherobots/db/errors.py,sha256=G8a-DB6pgB0pUld59Wyq8PuywL-JWxGJfvrr-g8BPuU,310
|
|
8
8
|
wherobots/db/models.py,sha256=uk0EqZMH_RFkQusSw2AvyFivGvGJBu_WYP8FHa6JJGM,4178
|
|
9
9
|
wherobots/db/redaction.py,sha256=issl5e5QvZQbs4d5UGUJIW7SIZIYfQ04MIyuT0Eo5_g,3744
|
|
10
10
|
wherobots/db/region.py,sha256=GoFzazvgOjE1kVVT1QKAsRK88c_1XSAebR3ME4foSlQ,400
|
|
11
11
|
wherobots/db/runtime.py,sha256=eAR1oJEtssgkp-IRPTLOrlPI-_5SM_ASw6T6hQ_qF4I,985
|
|
12
12
|
wherobots/db/session_type.py,sha256=jZNwCl0nxvY-0reIn8RR_-fIYSSvIcaGx_4pRc71Lww,137
|
|
13
|
-
wherobots/db/types.py,sha256=
|
|
14
|
-
wherobots_python_dbapi-0.
|
|
15
|
-
wherobots_python_dbapi-0.
|
|
16
|
-
wherobots_python_dbapi-0.
|
|
17
|
-
wherobots_python_dbapi-0.
|
|
13
|
+
wherobots/db/types.py,sha256=uRobm9S8CERnV0a3uZA5hJ1cLxZGZcIHFZRcRQ6KX10,2615
|
|
14
|
+
wherobots_python_dbapi-0.29.0.dist-info/METADATA,sha256=zyJNqMu87nw1nKnZ-3fUB0znHEJ6OSaxH-oyRT5lPtQ,11509
|
|
15
|
+
wherobots_python_dbapi-0.29.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
16
|
+
wherobots_python_dbapi-0.29.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
17
|
+
wherobots_python_dbapi-0.29.0.dist-info/RECORD,,
|
|
File without changes
|
{wherobots_python_dbapi-0.28.2.dist-info → wherobots_python_dbapi-0.29.0.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|