fennecdb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fennecdb/__init__.py +1481 -0
- fennecdb/builder.py +1867 -0
- fennecdb/langchain.py +307 -0
- fennecdb/llama_index.py +333 -0
- fennecdb/relay.py +83 -0
- fennecdb-0.2.0.dist-info/METADATA +301 -0
- fennecdb-0.2.0.dist-info/RECORD +9 -0
- fennecdb-0.2.0.dist-info/WHEEL +5 -0
- fennecdb-0.2.0.dist-info/top_level.txt +1 -0
fennecdb/__init__.py
ADDED
|
@@ -0,0 +1,1481 @@
|
|
|
1
|
+
"""fennecdb over HTTP.
|
|
2
|
+
|
|
3
|
+
A client for fennec-server's HTTP endpoint (`fennec-server --http <address>`), and on
|
|
4
|
+
top of it vector stores for LangChain (`fennecdb.langchain`) and LlamaIndex
|
|
5
|
+
(`fennecdb.llama_index`).
|
|
6
|
+
|
|
7
|
+
from fennecdb import Client
|
|
8
|
+
db = Client("http://127.0.0.1:8080", token="...")
|
|
9
|
+
db.query("get articles near embed $1 limit 5", [[0.1, 0.2, 0.3]])
|
|
10
|
+
|
|
11
|
+
The client is the standard library alone, as the server is: a statement is
|
|
12
|
+
one `POST /query`, several are one `POST /batch` under one lock.
|
|
13
|
+
|
|
14
|
+
`collection` starts the query builder, which writes the statement for you
|
|
15
|
+
-- every value a parameter, every name checked -- and makes the same text
|
|
16
|
+
the JavaScript, Go and .NET builders make of the same chain:
|
|
17
|
+
|
|
18
|
+
from fennecdb import or_
|
|
19
|
+
rows = (db.collection("articles").select("title")
|
|
20
|
+
.where("year", ">=", 2024)
|
|
21
|
+
.where(or_({"lang": "tr"}, {"tags": {"has": "rust"}}))
|
|
22
|
+
.near("embed", [0.1, 0.2, 0.3]).limit(5).rows())
|
|
23
|
+
|
|
24
|
+
`match` takes `highlight` and `snippet` marks in the select list, and any
|
|
25
|
+
read `facet` counts, which come back beside the rows as `rows.facets`:
|
|
26
|
+
|
|
27
|
+
rows = (db.collection("products").select("title").highlight("title")
|
|
28
|
+
.match("title", "phone").facet("brand", top=10).limit(20).rows())
|
|
29
|
+
rows.facets["brand"] # [FacetCount(value="acme", count=12), ...]
|
|
30
|
+
|
|
31
|
+
A named statement (`create statement transfer(...) as ...`) is called by
|
|
32
|
+
name with its arguments, an object, as one parameter -- `call transfer $1`
|
|
33
|
+
over `/query`, so the errors, the key and `wait` are a statement's; the
|
|
34
|
+
server's token may say the claims it is made with (`as $2`), and `read=True`
|
|
35
|
+
marks a statement that only reads, to go near a read replica:
|
|
36
|
+
|
|
37
|
+
db.call("transfer", {"from": 1, "to": 2, "amount": 300}, idempotency_key=transfer_id)
|
|
38
|
+
db.call("history", read=True)
|
|
39
|
+
text, params = call("transfer", {"from": 1, "to": 2, "amount": 300}).to_call()
|
|
40
|
+
|
|
41
|
+
A retry after a timeout cannot tell whether the write ran; under an
|
|
42
|
+
`Idempotency-Key` it runs once, and the second answer is the first one
|
|
43
|
+
kept (`replayed`). One key per write -- the same key with another request
|
|
44
|
+
is refused (422):
|
|
45
|
+
|
|
46
|
+
db.batch([debit, credit], idempotency_key=transfer_id)
|
|
47
|
+
db.with_idempotency_key(order_id).collection("orders").insert(order)
|
|
48
|
+
|
|
49
|
+
A worker's claim that finds no job can wait for one at the server
|
|
50
|
+
(`Fennec-Wait`): with `wait` a `set` or `del` that writes nothing is held
|
|
51
|
+
until it can -- a job enqueued, a delayed one come due, a lease lapsed --
|
|
52
|
+
or for that many seconds, at most 30, rather than sent again every few:
|
|
53
|
+
|
|
54
|
+
jobs = db.collection("jobs").where(raw("run_at <= now()")).order("run_at").limit(10)
|
|
55
|
+
claimed = jobs.update(claim, returning=True, wait=30)
|
|
56
|
+
|
|
57
|
+
`AsyncClient` is the same over asyncio, for an event loop a blocking
|
|
58
|
+
request would stall:
|
|
59
|
+
|
|
60
|
+
async with AsyncClient("http://127.0.0.1:8080", token="...") as db:
|
|
61
|
+
rows = await db.query("get articles near embed $1 limit 5", [[0.1, 0.2, 0.3]])
|
|
62
|
+
|
|
63
|
+
A client far from a tenant's home reads near itself where the tenant has a
|
|
64
|
+
read replica there (`fennec-shard`'s read replicas): with `reads="nearest"`
|
|
65
|
+
the builder's reads, and a statement given `read=True`, are marked
|
|
66
|
+
(`Fennec-Read: replica`) for a router near a replica to answer, each sent
|
|
67
|
+
after the furthest position an answer from home named to this client
|
|
68
|
+
(`Fennec-After` and `Fennec-History`: the change, and the history it is a
|
|
69
|
+
change of) -- it reads nothing older than it wrote, nor a write a failover
|
|
70
|
+
replaced. Writes go home:
|
|
71
|
+
|
|
72
|
+
db = Client("https://us.example.com/t/acme", token="...", reads="nearest")
|
|
73
|
+
|
|
74
|
+
`Client.follow` reads every write on the server's disk (`GET /_changes`,
|
|
75
|
+
`fennec-server --cdc`) for a consumer whose cursor the server keeps, and
|
|
76
|
+
commits each batch once the loop comes back for the next:
|
|
77
|
+
|
|
78
|
+
for batch in db.follow("search-index"):
|
|
79
|
+
for change in batch:
|
|
80
|
+
index(change) # an exception: this batch is read again
|
|
81
|
+
|
|
82
|
+
`python -m fennecdb.relay` hands the same to another program or a webhook.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
from __future__ import annotations
|
|
86
|
+
|
|
87
|
+
import asyncio
|
|
88
|
+
import errno
|
|
89
|
+
import http.client
|
|
90
|
+
import json
|
|
91
|
+
import random
|
|
92
|
+
import re
|
|
93
|
+
import select
|
|
94
|
+
import socket
|
|
95
|
+
import ssl
|
|
96
|
+
import threading
|
|
97
|
+
import time
|
|
98
|
+
import urllib.parse
|
|
99
|
+
from datetime import datetime
|
|
100
|
+
from typing import Any, Iterable, Iterator, Mapping, NamedTuple, Sequence
|
|
101
|
+
|
|
102
|
+
__all__ = [
|
|
103
|
+
"AsyncClient",
|
|
104
|
+
"AsyncQuery",
|
|
105
|
+
"Call",
|
|
106
|
+
"Changes",
|
|
107
|
+
"Client",
|
|
108
|
+
"Computed",
|
|
109
|
+
"Cond",
|
|
110
|
+
"FacetCount",
|
|
111
|
+
"FennecError",
|
|
112
|
+
"HeldSlot",
|
|
113
|
+
"LastRun",
|
|
114
|
+
"NamedStatement",
|
|
115
|
+
"Param",
|
|
116
|
+
"Query",
|
|
117
|
+
"Retry",
|
|
118
|
+
"Rows",
|
|
119
|
+
"SchemaError",
|
|
120
|
+
"and_",
|
|
121
|
+
"bucket",
|
|
122
|
+
"call",
|
|
123
|
+
"collection",
|
|
124
|
+
"count_distinct",
|
|
125
|
+
"distance",
|
|
126
|
+
"expr",
|
|
127
|
+
"first",
|
|
128
|
+
"inc",
|
|
129
|
+
"last",
|
|
130
|
+
"not_",
|
|
131
|
+
"or_",
|
|
132
|
+
"param",
|
|
133
|
+
"placeholders",
|
|
134
|
+
"raw",
|
|
135
|
+
]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class FennecError(Exception):
|
|
139
|
+
"""A statement the server refused: its message, and the HTTP status --
|
|
140
|
+
0 for one the query builder refused before sending anything. A batch
|
|
141
|
+
that stopped also says which statement stopped it, from 0 (`at`: the
|
|
142
|
+
write whose `require` was not met, 412, or whose id was taken, 409),
|
|
143
|
+
and how many statements stayed applied (`completed`: 0, since a batch
|
|
144
|
+
lands whole, but for one holding a `compact`); both None otherwise.
|
|
145
|
+
|
|
146
|
+
A `require n else '<reason>'` that refused says its reason (`reason`,
|
|
147
|
+
the builders' `reason=`): the statement's own words, by which a caller
|
|
148
|
+
tells "insufficient funds" from "no such account" -- where `at` moves
|
|
149
|
+
with every statement added in front of it. None where the statement
|
|
150
|
+
gave none.
|
|
151
|
+
|
|
152
|
+
A statement bound to the write (`create statement ... after`) that
|
|
153
|
+
refused it is named (`binding`): a guard's `require` not met, a write
|
|
154
|
+
of its body refused. Told the server's token alone, a scoped one the
|
|
155
|
+
`reason` and no name; None otherwise.
|
|
156
|
+
|
|
157
|
+
A tenant's quota names itself (`quota`): "bytes" and "rows" answer 507,
|
|
158
|
+
and room is made by deleting; "streams", "held", "requests", "writes"
|
|
159
|
+
and "write_bytes" answer 429, with the seconds to wait before sending
|
|
160
|
+
again (`retry_after`) -- to the millisecond, as the server says it in
|
|
161
|
+
`retry_after_ms` -- as a tenant being moved or failed over answers its
|
|
162
|
+
writes 503, in whole seconds. None otherwise.
|
|
163
|
+
|
|
164
|
+
A 500 whose `outcome` is "unknown" is a write a tenant's node landed and
|
|
165
|
+
answered after its lease from the router lapsed: its tenant may have
|
|
166
|
+
been failed over without it. Send it again with the same idempotency
|
|
167
|
+
key, or read it back a lease and a tenth later; `seq` and `history` say
|
|
168
|
+
where it left the node's database, and a read sent after them once the
|
|
169
|
+
tenant settles is answered by a primary holding the write and refused
|
|
170
|
+
504 by one without it. Not the client's position, which this answer
|
|
171
|
+
does not move. None otherwise."""
|
|
172
|
+
|
|
173
|
+
def __init__(
|
|
174
|
+
self,
|
|
175
|
+
message: str,
|
|
176
|
+
status: int,
|
|
177
|
+
*,
|
|
178
|
+
at: int | None = None,
|
|
179
|
+
completed: int | None = None,
|
|
180
|
+
quota: str | None = None,
|
|
181
|
+
retry_after: float | None = None,
|
|
182
|
+
outcome: str | None = None,
|
|
183
|
+
seq: int | None = None,
|
|
184
|
+
history: str | None = None,
|
|
185
|
+
reason: str | None = None,
|
|
186
|
+
binding: str | None = None,
|
|
187
|
+
):
|
|
188
|
+
super().__init__(message)
|
|
189
|
+
self.status = status
|
|
190
|
+
self.at = at
|
|
191
|
+
self.completed = completed
|
|
192
|
+
self.reason = reason
|
|
193
|
+
self.binding = binding
|
|
194
|
+
self.quota = quota
|
|
195
|
+
self.retry_after = retry_after
|
|
196
|
+
self.outcome = outcome
|
|
197
|
+
self.seq = seq
|
|
198
|
+
self.history = history
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class SchemaError(FennecError):
|
|
202
|
+
"""The schema the code declares and the server's differ in what no
|
|
203
|
+
open applies: each difference and how to resolve it, as the engine
|
|
204
|
+
writes them, in `refusals`."""
|
|
205
|
+
|
|
206
|
+
def __init__(self, refusals: list, status: int):
|
|
207
|
+
self.refusals = refusals
|
|
208
|
+
lines = [f" - {r['message']}\n {r['fix']}" for r in refusals]
|
|
209
|
+
super().__init__("the database's schema differs from the code's:\n" + "\n".join(lines), status)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class Changes(NamedTuple):
|
|
213
|
+
"""Writes `GET /_changes` handed over, a dict each, and `next`: the last
|
|
214
|
+
one's number, which is the `since` to read on from. `seq` is the last
|
|
215
|
+
write the database holds (`Fennec-Seq`): a `next` that has reached it has
|
|
216
|
+
every write there is."""
|
|
217
|
+
|
|
218
|
+
writes: list
|
|
219
|
+
next: int
|
|
220
|
+
seq: int = 0
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class LastRun(NamedTuple):
|
|
224
|
+
"""A scheduled statement's last run, landed or pending, as `statements`
|
|
225
|
+
lists it: the slot it ran for (what `slot()` read), `"ran"` or
|
|
226
|
+
`"refused"` -- the statement's own refusal, a `require` not met or a
|
|
227
|
+
broken statement, which marks the slot as run -- its status and the
|
|
228
|
+
`reason` its `require` gave, and the slots passed over before it."""
|
|
229
|
+
|
|
230
|
+
slot: datetime
|
|
231
|
+
outcome: str
|
|
232
|
+
status: int | None
|
|
233
|
+
reason: str | None
|
|
234
|
+
skipped: int
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class HeldSlot(NamedTuple):
|
|
238
|
+
"""A due slot the server's state holds back -- a tenant's quota (507), a
|
|
239
|
+
storage error -- tried again as that changes, and since when."""
|
|
240
|
+
|
|
241
|
+
slot: datetime
|
|
242
|
+
status: int
|
|
243
|
+
since: datetime
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class NamedStatement(NamedTuple):
|
|
247
|
+
"""A named statement as `statements` lists it to the server's token,
|
|
248
|
+
each column None where it says nothing of the statement: a called one
|
|
249
|
+
has no `after` and no clock. `after` is a write-bound statement's
|
|
250
|
+
collection and the writes that run it (`("orders", "set or del")`);
|
|
251
|
+
`schedule` a scheduled one's clause as its text writes it (`every 1h`,
|
|
252
|
+
`at '0 3 * * *' in 'Europe/Berlin'`), `next` the slot due next -- past
|
|
253
|
+
while one is overdue or held -- `last` its last run and `held` a slot
|
|
254
|
+
held back. `hook` is a webhook consumer's hook, whose events are sent
|
|
255
|
+
to `/_hooks/<hook>`, and `armed` whether this server -- on a tenant
|
|
256
|
+
node, the tenant -- holds its secret: `False` answers every event of
|
|
257
|
+
it 401. `version` is the history and change its definition landed
|
|
258
|
+
at; `broken` why the schema broke it, which it runs no more until it
|
|
259
|
+
reads again."""
|
|
260
|
+
|
|
261
|
+
name: str
|
|
262
|
+
version: tuple[int, int]
|
|
263
|
+
signature: str | None
|
|
264
|
+
writes: bool | None
|
|
265
|
+
comment: str | None
|
|
266
|
+
broken: str | None
|
|
267
|
+
after: tuple[str, str] | None
|
|
268
|
+
schedule: str | None
|
|
269
|
+
next: datetime | None
|
|
270
|
+
last: LastRun | None
|
|
271
|
+
held: HeldSlot | None
|
|
272
|
+
hook: str | None = None
|
|
273
|
+
armed: bool | None = None
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _instant(text: str) -> datetime:
|
|
277
|
+
"""The server's ISO text of a time, read as an aware UTC datetime:
|
|
278
|
+
`fromisoformat` takes a `Z` only from Python 3.11."""
|
|
279
|
+
return datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _named(row: Mapping[str, Any]) -> NamedStatement:
|
|
283
|
+
"""A row of `statements` as a `NamedStatement`."""
|
|
284
|
+
last, held, after = row.get("last"), row.get("held"), row.get("after")
|
|
285
|
+
return NamedStatement(
|
|
286
|
+
name=row["name"],
|
|
287
|
+
version=tuple(row["version"]),
|
|
288
|
+
signature=row.get("signature"),
|
|
289
|
+
writes=row.get("writes"),
|
|
290
|
+
comment=row.get("comment"),
|
|
291
|
+
broken=row.get("broken"),
|
|
292
|
+
after=tuple(after) if after else None,
|
|
293
|
+
schedule=row.get("schedule"),
|
|
294
|
+
next=_instant(row["next"]) if row.get("next") else None,
|
|
295
|
+
last=(
|
|
296
|
+
LastRun(_instant(last["slot"]), last["outcome"], last.get("status"), last.get("reason"), last.get("skipped", 0))
|
|
297
|
+
if last
|
|
298
|
+
else None
|
|
299
|
+
),
|
|
300
|
+
held=HeldSlot(_instant(held["slot"]), held["status"], _instant(held["since"])) if held else None,
|
|
301
|
+
hook=row.get("hook"),
|
|
302
|
+
armed=row.get("armed"),
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class Retry(NamedTuple):
|
|
307
|
+
"""How a client sends a request again: only where nothing ran, or where
|
|
308
|
+
running it again changes nothing.
|
|
309
|
+
|
|
310
|
+
Nothing ran: a 503, or a quota's 429, naming its wait -- sent after that
|
|
311
|
+
wait, never sooner, unless its body says statements of a batch were
|
|
312
|
+
made (`completed`), which the server never answers so; a connection
|
|
313
|
+
refused; a request not handed to the system whole. Running it again
|
|
314
|
+
changes nothing: a read the builder made, one sent near (`read=True` on
|
|
315
|
+
a client with `reads="nearest"`: `Fennec-Read: replica`, which every
|
|
316
|
+
server runs read-only), and a write under an `Idempotency-Key`, answered
|
|
317
|
+
from its key while the server keeps it (a day) -- these go again after a
|
|
318
|
+
reset or a dropped connection too. Nothing goes again after the client's
|
|
319
|
+
own `timeout`, past which the server may still run it, nor a refusal
|
|
320
|
+
naming no wait; `read=True` sent home is a claim nothing checks, and
|
|
321
|
+
goes once.
|
|
322
|
+
|
|
323
|
+
`attempts` tries at most, their waits -- the server's, or 0.25 s doubling,
|
|
324
|
+
30% jitter on top -- within `max_wait` seconds; past either, the last
|
|
325
|
+
answer or error is the caller's. `Client(retry=False)` sends nothing
|
|
326
|
+
again."""
|
|
327
|
+
|
|
328
|
+
attempts: int = 3
|
|
329
|
+
max_wait: float = 10.0
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _retry(retry: Retry | bool | None) -> Retry | None:
|
|
333
|
+
if retry is True:
|
|
334
|
+
return Retry()
|
|
335
|
+
if not retry:
|
|
336
|
+
return None
|
|
337
|
+
if not isinstance(retry, Retry) or retry.attempts < 1 or retry.max_wait < 0:
|
|
338
|
+
raise ValueError(f"bad retry: {retry!r}")
|
|
339
|
+
return retry
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _wait_of(status: int, raw: bytes, header: str | None) -> float | None:
|
|
343
|
+
"""The seconds a 503 or a 429 asks a client to wait, where it names any:
|
|
344
|
+
then nothing ran. To the millisecond where the body says it
|
|
345
|
+
(`retry_after_ms`), before `Retry-After` -- whole seconds -- and the
|
|
346
|
+
body's `retry_after`, which every 503 of the server's carries. None
|
|
347
|
+
where statements of a batch were made (`completed`): sent again whole,
|
|
348
|
+
they were made twice."""
|
|
349
|
+
if status not in (429, 503):
|
|
350
|
+
return None
|
|
351
|
+
try:
|
|
352
|
+
body = json.loads(raw)
|
|
353
|
+
except ValueError:
|
|
354
|
+
body = None
|
|
355
|
+
if not isinstance(body, dict):
|
|
356
|
+
body = {}
|
|
357
|
+
done = body.get("completed")
|
|
358
|
+
if isinstance(done, int) and done > 0:
|
|
359
|
+
return None
|
|
360
|
+
ms = body.get("retry_after_ms")
|
|
361
|
+
if isinstance(ms, (int, float)) and not isinstance(ms, bool) and ms > 0:
|
|
362
|
+
return ms / 1000
|
|
363
|
+
for said in (header, body.get("retry_after")):
|
|
364
|
+
try:
|
|
365
|
+
wait = float(said)
|
|
366
|
+
except (TypeError, ValueError):
|
|
367
|
+
continue
|
|
368
|
+
if 0 < wait < float("inf"):
|
|
369
|
+
return wait
|
|
370
|
+
return None
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _pause(retry: Retry | None, tries: int, waited: float, wait: float) -> float | None:
|
|
374
|
+
"""How long to wait before try `tries + 1`, after `waited` seconds of
|
|
375
|
+
waits: the server's `wait`, or the backoff where longer, 30% jitter on
|
|
376
|
+
top, within what is left of `max_wait`; None where no try is left."""
|
|
377
|
+
base = max(wait, 0.25 * 2 ** (tries - 1))
|
|
378
|
+
if retry is None or tries >= retry.attempts or waited + base > retry.max_wait:
|
|
379
|
+
return None
|
|
380
|
+
return min(base * (1 + random.random() * 0.3), retry.max_wait - waited)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _bound(sock: socket.socket, secs: float | None) -> None:
|
|
384
|
+
"""Gives each of `sock`'s next waits `secs`, None for no bound, and
|
|
385
|
+
gives the request up where none is left. Within a millisecond either
|
|
386
|
+
way: a `settimeout` is a system call (0.3 us), and the one a quick
|
|
387
|
+
request left is as good as set."""
|
|
388
|
+
if secs is not None and secs <= 0:
|
|
389
|
+
raise TimeoutError("timed out")
|
|
390
|
+
was = sock.gettimeout()
|
|
391
|
+
if (was is None) != (secs is None) or (secs is not None and abs(was - secs) > 0.001):
|
|
392
|
+
sock.settimeout(secs)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# One refusal in an OSError's words, as asyncio joins every address's
|
|
396
|
+
# error when a host's addresses all fail ("Multiple exceptions: ...").
|
|
397
|
+
_REFUSAL = re.compile(rf"\[(?:Errno {errno.ECONNREFUSED}|WinError 1225)\] [^\[]*")
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _refused(e: BaseException) -> bool:
|
|
401
|
+
"""Whether connecting was refused, at every address of the host's where
|
|
402
|
+
it has several: `asyncio.open_connection` tries each and, their words
|
|
403
|
+
differing by the address, raises one `OSError` joining them, the
|
|
404
|
+
refusals in it no `ConnectionRefusedError` any more. A request refused
|
|
405
|
+
so sent nothing, and any goes again."""
|
|
406
|
+
if isinstance(e, ConnectionRefusedError):
|
|
407
|
+
return True
|
|
408
|
+
said = str(e)
|
|
409
|
+
rest = said.removeprefix("Multiple exceptions: ")
|
|
410
|
+
if not isinstance(e, OSError) or rest == said:
|
|
411
|
+
return False
|
|
412
|
+
return all(_REFUSAL.fullmatch(part) for part in re.split(r", (?=\[(?:Errno|WinError) )", rest))
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class _Shared:
|
|
416
|
+
"""What a client and the copies `with_idempotency_key` makes of it
|
|
417
|
+
share, as Go's and .NET's copies share theirs: the change the last
|
|
418
|
+
answer from home left the database at, whether it was replayed, and
|
|
419
|
+
the furthest position an answer from home named, which a read near
|
|
420
|
+
waits for -- an answer that arrives late is not the last."""
|
|
421
|
+
|
|
422
|
+
__slots__ = ("seq", "replayed", "at", "_lock")
|
|
423
|
+
|
|
424
|
+
def __init__(self) -> None:
|
|
425
|
+
self.seq: int | None = None
|
|
426
|
+
self.replayed = False
|
|
427
|
+
self.at: tuple | None = None
|
|
428
|
+
# A client's threads share its copies' position.
|
|
429
|
+
self._lock = threading.Lock()
|
|
430
|
+
|
|
431
|
+
def answered(self, seq: int, history: str | None) -> None:
|
|
432
|
+
"""An answer from home named `seq`, of `history`."""
|
|
433
|
+
p = _position(seq, history)
|
|
434
|
+
with self._lock:
|
|
435
|
+
self.seq = seq
|
|
436
|
+
if _later(p, self.at):
|
|
437
|
+
self.at = p
|
|
438
|
+
|
|
439
|
+
def told(self, raw: bytes, sent: tuple) -> None:
|
|
440
|
+
"""A read near's 504, the read sent after `sent`: no copy held it,
|
|
441
|
+
home either -- a write lost in a failover. Home's position, lower or
|
|
442
|
+
not, is the client's from now on: kept, every read near would wait
|
|
443
|
+
it out for good. Taken only over `sent` -- a write answered
|
|
444
|
+
meanwhile is further, and set back the next read near would miss
|
|
445
|
+
it -- and never of an earlier term of the same lineage: a router
|
|
446
|
+
whose copy of the directory lags a failover sends the read to the
|
|
447
|
+
old primary, fenced, which names the branch it abandoned. Another
|
|
448
|
+
lineage's is a tenant made again under the name, which starts again
|
|
449
|
+
at term 1."""
|
|
450
|
+
try:
|
|
451
|
+
body = json.loads(raw)
|
|
452
|
+
except ValueError:
|
|
453
|
+
return
|
|
454
|
+
if isinstance(body, dict) and isinstance(body.get("seq"), int):
|
|
455
|
+
history = body.get("history")
|
|
456
|
+
p = _position(body["seq"], history if isinstance(history, str) else None)
|
|
457
|
+
with self._lock:
|
|
458
|
+
if self.at is sent and (p[0] >= sent[0] or _another(p, sent)):
|
|
459
|
+
self.at = p
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _position(seq: int, history: str | None) -> tuple:
|
|
463
|
+
"""`(term, seq, history)`: a change, the history it is of
|
|
464
|
+
(`<term>-<id>`; none for a database that never had a replica) and that
|
|
465
|
+
history's term, one more at each promotion."""
|
|
466
|
+
term = 0
|
|
467
|
+
if history:
|
|
468
|
+
try:
|
|
469
|
+
term = int(history.split("-", 1)[0])
|
|
470
|
+
except ValueError:
|
|
471
|
+
term = 0
|
|
472
|
+
return (term, seq, history)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _lineage(p: tuple) -> str | None:
|
|
476
|
+
"""The lineage of position `p`: its history id's top half, which every
|
|
477
|
+
fork of the tenant keeps and a tenant made again under its name does
|
|
478
|
+
not."""
|
|
479
|
+
history = p[2]
|
|
480
|
+
if not history or "-" not in history:
|
|
481
|
+
return None
|
|
482
|
+
return history.split("-", 1)[1][:8] or None
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _another(p: tuple, q: tuple) -> bool:
|
|
486
|
+
"""Whether `p` is of another lineage than `q`: a tenant made again."""
|
|
487
|
+
a, b = _lineage(p), _lineage(q)
|
|
488
|
+
return a is not None and b is not None and a != b
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _later(p: tuple, q: tuple | None) -> bool:
|
|
492
|
+
"""Whether position `p` is past `q`: of another lineage -- home's word
|
|
493
|
+
on a tenant made again, whose terms start again -- or a later term of
|
|
494
|
+
the same, or further in one."""
|
|
495
|
+
return q is None or _another(p, q) or p[:2] > q[:2]
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _reads(reads: str) -> str:
|
|
499
|
+
if reads not in ("home", "nearest"):
|
|
500
|
+
raise ValueError(f"reads is 'home' or 'nearest' (got {reads!r})")
|
|
501
|
+
return reads
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _near(headers: dict, at: tuple | None, explicit: int | None) -> None:
|
|
505
|
+
"""A read marked for a replica near the client, after `explicit` where
|
|
506
|
+
the caller named a change, and after its furthest position otherwise."""
|
|
507
|
+
headers["Fennec-Read"] = "replica"
|
|
508
|
+
if explicit is not None:
|
|
509
|
+
headers["Fennec-After"] = str(explicit)
|
|
510
|
+
elif at is not None:
|
|
511
|
+
headers["Fennec-After"] = str(at[1])
|
|
512
|
+
if at[2]:
|
|
513
|
+
headers["Fennec-History"] = at[2]
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
class Client:
|
|
517
|
+
"""fennec-server's HTTP endpoint, one statement a request.
|
|
518
|
+
|
|
519
|
+
A statement goes over a connection kept alive, one a thread: through
|
|
520
|
+
`urlopen`, which opens one a request, a read by id took 152 us against
|
|
521
|
+
57 over the one connection, the server starting a thread for each.
|
|
522
|
+
|
|
523
|
+
`timeout` bounds the seconds to an answer's head -- a `wait` holds it
|
|
524
|
+
that much longer -- and those between the pieces of its body, which is
|
|
525
|
+
read for as long as it keeps coming; None, never. A statement that runs
|
|
526
|
+
long before it answers -- a `compact`, a `create index` -- is given a
|
|
527
|
+
client with a longer `timeout`: past it, it may still run, and is never
|
|
528
|
+
sent again. `retry` sends a request again where nothing ran or running
|
|
529
|
+
it twice is harmless (`Retry`); `False`, never.
|
|
530
|
+
"""
|
|
531
|
+
|
|
532
|
+
def __init__(
|
|
533
|
+
self,
|
|
534
|
+
url: str = "http://127.0.0.1:8080",
|
|
535
|
+
token: str | None = None,
|
|
536
|
+
timeout: float | None = 30.0,
|
|
537
|
+
reads: str = "home",
|
|
538
|
+
retry: Retry | bool = True,
|
|
539
|
+
):
|
|
540
|
+
parts = urllib.parse.urlsplit(url)
|
|
541
|
+
if parts.scheme not in ("http", "https"):
|
|
542
|
+
raise ValueError(f"not an http(s) URL: {url!r}")
|
|
543
|
+
self.url = url.rstrip("/")
|
|
544
|
+
self.token = token
|
|
545
|
+
self.timeout = timeout
|
|
546
|
+
self.retry = _retry(retry)
|
|
547
|
+
# "nearest": reads marked for a read replica near the client.
|
|
548
|
+
self.reads = _reads(reads)
|
|
549
|
+
self.idempotency_key: str | None = None
|
|
550
|
+
self._shared = _Shared()
|
|
551
|
+
self._host = parts.hostname or "127.0.0.1"
|
|
552
|
+
self._port = parts.port or (443 if parts.scheme == "https" else 80)
|
|
553
|
+
self._tls = parts.scheme == "https"
|
|
554
|
+
self._base = parts.path.rstrip("/")
|
|
555
|
+
self._local = threading.local()
|
|
556
|
+
|
|
557
|
+
@property
|
|
558
|
+
def seq(self) -> int | None:
|
|
559
|
+
"""The change the last write through this client, or a copy of it,
|
|
560
|
+
left the database at (`Fennec-Seq`): a read on a replica passed it as
|
|
561
|
+
`after` waits for that write."""
|
|
562
|
+
return self._shared.seq
|
|
563
|
+
|
|
564
|
+
@seq.setter
|
|
565
|
+
def seq(self, value: int | None) -> None:
|
|
566
|
+
self._shared.seq = value
|
|
567
|
+
|
|
568
|
+
@property
|
|
569
|
+
def replayed(self) -> bool:
|
|
570
|
+
"""Whether the last write's answer was the one kept for its
|
|
571
|
+
`Idempotency-Key`: the write had run before, and did not again."""
|
|
572
|
+
return self._shared.replayed
|
|
573
|
+
|
|
574
|
+
def with_idempotency_key(self, key: str) -> "Client":
|
|
575
|
+
"""A copy of the client whose writes -- `query`, `batch` and the
|
|
576
|
+
builder's -- carry `key` as their `Idempotency-Key`: sent again
|
|
577
|
+
after a timeout, a write runs once. It shares this client's
|
|
578
|
+
connections and `seq`. One key per write; the same key with
|
|
579
|
+
another request is refused (422)."""
|
|
580
|
+
if not isinstance(key, str) or not key:
|
|
581
|
+
raise ValueError("an idempotency key is a text, not empty")
|
|
582
|
+
copy = object.__new__(Client)
|
|
583
|
+
copy.__dict__.update(self.__dict__)
|
|
584
|
+
copy.idempotency_key = key
|
|
585
|
+
return copy
|
|
586
|
+
|
|
587
|
+
def collection(self, name: str) -> Query:
|
|
588
|
+
"""The query builder over collection `name`: chain `select`, `where`,
|
|
589
|
+
`near`, `order`, `limit` ... and end with `rows()`, `first()`,
|
|
590
|
+
`count()`, or a write -- `insert`, `update`, `delete`."""
|
|
591
|
+
return Query(name, self)
|
|
592
|
+
|
|
593
|
+
def call(
|
|
594
|
+
self,
|
|
595
|
+
name: str,
|
|
596
|
+
args: Mapping[str, Any] | None = None,
|
|
597
|
+
*,
|
|
598
|
+
claims: Mapping[str, Any] | None = None,
|
|
599
|
+
idempotency_key: str | None = None,
|
|
600
|
+
wait: float | None = None,
|
|
601
|
+
read: bool = False,
|
|
602
|
+
) -> Any:
|
|
603
|
+
"""Calls the named statement `name` with `args`, a mapping of values
|
|
604
|
+
by name: `call <name> $1` over `/query`, sent and answered as
|
|
605
|
+
`query` does it -- the last statement of its body, rows or
|
|
606
|
+
`{"affected": n}`. A refusal is a `FennecError` as any statement's:
|
|
607
|
+
`status` 404 for no such statement or none granted, 403 for a claim
|
|
608
|
+
the token lacks, 400 for an argument missing, extra or of another
|
|
609
|
+
type, 412 for a `require` not met (its `reason`), 409 for a unique
|
|
610
|
+
value taken or a broken statement. `claims` is `as {...}`, the
|
|
611
|
+
claims the call is made with: the server's token alone may say them,
|
|
612
|
+
a scoped token's are its own. `idempotency_key` and `wait` are
|
|
613
|
+
`query`'s. `read=True` says the statement only reads, and on a
|
|
614
|
+
client with `reads="nearest"` it goes near, after this client's
|
|
615
|
+
writes; one that writes is refused there (403), never landed. Not
|
|
616
|
+
said, a call goes home, as every text not known to read does."""
|
|
617
|
+
text, params = Call(name, args, claims=claims).to_call()
|
|
618
|
+
return self.query(text, params, idempotency_key=idempotency_key, wait=wait, read=read)
|
|
619
|
+
|
|
620
|
+
def statements(self) -> list[NamedStatement]:
|
|
621
|
+
"""Each named statement the database holds, as the server lists them
|
|
622
|
+
to its own token (`statements`) -- a scoped one is refused (403):
|
|
623
|
+
its signature, whether it writes, its comment, why the schema broke
|
|
624
|
+
it, what a binding is bound to, a scheduled one's clock, and a
|
|
625
|
+
webhook consumer's hook and whether the server holds its secret.
|
|
626
|
+
`call` sends a webhook consumer a test event, the event its
|
|
627
|
+
argument, run as a delivery is and keeping no key."""
|
|
628
|
+
return [_named(r) for r in self.query("statements")]
|
|
629
|
+
|
|
630
|
+
def query(
|
|
631
|
+
self,
|
|
632
|
+
fennecql: str,
|
|
633
|
+
params: Sequence[Any] | None = None,
|
|
634
|
+
*,
|
|
635
|
+
after: int | None = None,
|
|
636
|
+
idempotency_key: str | None = None,
|
|
637
|
+
wait: float | None = None,
|
|
638
|
+
read: bool = False,
|
|
639
|
+
) -> Any:
|
|
640
|
+
"""Runs one FennecQL statement. Values go in as `$1`, `$2`, ... and
|
|
641
|
+
never into the text. With `after` -- a primary's `seq` -- a replica
|
|
642
|
+
answers once it holds that write. With `idempotency_key` a write
|
|
643
|
+
runs once however often it is sent (`replayed`). With `wait` a
|
|
644
|
+
`set` or `del` that writes nothing is held at the server until it
|
|
645
|
+
can, or that many seconds -- a claim waiting for a job. `read=True`
|
|
646
|
+
says the statement only reads: on a client with `reads="nearest"`,
|
|
647
|
+
a router near the tenant's read replica answers it there, after
|
|
648
|
+
this client's writes. A read answers its rows, a list; one with a
|
|
649
|
+
`facet` clause a `Rows`, the counts as its `facets`."""
|
|
650
|
+
body = {"query": fennecql, "params": list(params or [])}
|
|
651
|
+
return _faceted(
|
|
652
|
+
self._post(
|
|
653
|
+
"/query",
|
|
654
|
+
json.dumps(body).encode(),
|
|
655
|
+
"application/json",
|
|
656
|
+
after,
|
|
657
|
+
key=idempotency_key,
|
|
658
|
+
wait=wait,
|
|
659
|
+
read=read,
|
|
660
|
+
)
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
def batch(
|
|
664
|
+
self,
|
|
665
|
+
statements: Iterable[tuple[str, Sequence[Any]]],
|
|
666
|
+
*,
|
|
667
|
+
idempotency_key: str | None = None,
|
|
668
|
+
wait: float | None = None,
|
|
669
|
+
read: bool = False,
|
|
670
|
+
) -> Any:
|
|
671
|
+
"""Runs statements in order under one write lock, as one block:
|
|
672
|
+
their writes -- a create, a drop or a `create index` among them --
|
|
673
|
+
all land or, at the first error, none of them do: a `FennecError`
|
|
674
|
+
whose `at` is the statement that stopped it. A batch holding a
|
|
675
|
+
`compact` runs each statement on its own, and what ran before an
|
|
676
|
+
error stays (`completed`). Answers `{"ok": n, "results": [...]}`,
|
|
677
|
+
a statement's `{"affected": n}` or `{"rows": [...]}` each, and
|
|
678
|
+
leaves the change it wrote at in `seq`. With `idempotency_key` it
|
|
679
|
+
lands once however often it is sent (`replayed`); a batch of reads
|
|
680
|
+
alone takes no key. With `wait` one whose `set`s and `del`s write
|
|
681
|
+
nothing is held until they can, as `query`'s; `read=True`, a batch
|
|
682
|
+
of reads alone, goes near as `query`'s does."""
|
|
683
|
+
lines = [json.dumps({"query": q, "params": list(p)}) for q, p in statements]
|
|
684
|
+
return self._post(
|
|
685
|
+
"/batch",
|
|
686
|
+
"\n".join(lines).encode(),
|
|
687
|
+
"application/x-ndjson",
|
|
688
|
+
key=idempotency_key,
|
|
689
|
+
wait=wait,
|
|
690
|
+
read=read,
|
|
691
|
+
)
|
|
692
|
+
|
|
693
|
+
def schema(
|
|
694
|
+
self, fennecql: str, migrations: Sequence[Any] | None = None, *, migrate: bool = False
|
|
695
|
+
) -> dict:
|
|
696
|
+
"""Checks the server's schema against `fennecql` -- `create
|
|
697
|
+
collection` and `create index` statements, a `schema.fennecql` file
|
|
698
|
+
-- and returns the engine's plan. The server owns its schema: it is
|
|
699
|
+
compared, and nothing applied, unless `migrate` -- with the server's
|
|
700
|
+
token -- runs the migrations it has not recorded, in order, and adds
|
|
701
|
+
what only adds, all one block. Raises `SchemaError` naming every
|
|
702
|
+
difference that would lose data or could mean two things."""
|
|
703
|
+
body = {"format": 1, "fennecql": fennecql, "migrations": list(migrations or [])}
|
|
704
|
+
path = "/_schema/apply" if migrate else "/_schema/plan?mode=follow"
|
|
705
|
+
out = self._post(path, json.dumps(body).encode(), "application/json", accept=409)
|
|
706
|
+
if out["refusals"]:
|
|
707
|
+
raise SchemaError(out["refusals"], 409)
|
|
708
|
+
return out
|
|
709
|
+
|
|
710
|
+
def changes(
|
|
711
|
+
self,
|
|
712
|
+
since: int | None = None,
|
|
713
|
+
*,
|
|
714
|
+
consumer: str | None = None,
|
|
715
|
+
limit: int | None = None,
|
|
716
|
+
wait: float | None = None,
|
|
717
|
+
) -> Changes:
|
|
718
|
+
"""The writes on the server's disk after `since` -- or after where
|
|
719
|
+
`consumer` stands, or from now with neither -- at most `limit`, each
|
|
720
|
+
`{"seq", "at", "collection", "op", "id", "doc"}`. With `wait`, an
|
|
721
|
+
answer with nothing in it waits that many seconds for a write."""
|
|
722
|
+
query = {"since": since, "consumer": consumer, "limit": limit}
|
|
723
|
+
if wait is not None:
|
|
724
|
+
query["wait"] = int(wait * 1000)
|
|
725
|
+
q = urllib.parse.urlencode({k: v for k, v in query.items() if v is not None})
|
|
726
|
+
status, got, raw = self._send(
|
|
727
|
+
"GET", f"{self._base}/_changes?{q}", None, self._auth(), _more(self.timeout, wait), True
|
|
728
|
+
)
|
|
729
|
+
if not 200 <= status < 300:
|
|
730
|
+
_answer(status, raw)
|
|
731
|
+
nxt = int(got.get("Fennec-Next", since or 0))
|
|
732
|
+
seq = max(int(got.get("Fennec-Seq", 0)), nxt)
|
|
733
|
+
return Changes([json.loads(line) for line in raw.splitlines() if line], nxt, seq)
|
|
734
|
+
|
|
735
|
+
def consumer(self, name: str, since: int | None = None) -> dict:
|
|
736
|
+
"""Makes `name` a consumer at the last write on disk, or at `since`,
|
|
737
|
+
and returns where it stands, `{"name", "since"}`. One that is there
|
|
738
|
+
already stays where it stands -- `since` is a new consumer's start
|
|
739
|
+
-- so a program can call this every time it starts and go on from
|
|
740
|
+
where it stopped; a `since` ahead of where it stands is a
|
|
741
|
+
`FennecError` (409), a commit sent the old way. `move_consumer`
|
|
742
|
+
moves one."""
|
|
743
|
+
body = json.dumps({} if since is None else {"since": since}).encode()
|
|
744
|
+
return self._post(f"/_changes/consumers/{_seg(name)}", body, "application/json")
|
|
745
|
+
|
|
746
|
+
def move_consumer(self, name: str, since: int) -> dict:
|
|
747
|
+
"""Moves `name` to `since`: on, once it has done with every write up
|
|
748
|
+
to it (a commit), or back, to read from there again, and returns
|
|
749
|
+
where it stands after -- where it was, for a commit past nothing but
|
|
750
|
+
consumers' own commits, which reads the same writes. A consumer that
|
|
751
|
+
is not there is a `FennecError` (404): `consumer` makes one."""
|
|
752
|
+
body = json.dumps({"since": since}).encode()
|
|
753
|
+
headers = {**self._auth(), "Content-Type": "application/json"}
|
|
754
|
+
status, _, raw = self._send("PATCH", f"{self._base}/_changes/consumers/{_seg(name)}", body, headers)
|
|
755
|
+
return _answer(status, raw)
|
|
756
|
+
|
|
757
|
+
def consumers(self) -> list:
|
|
758
|
+
"""Each consumer: its `name`, `since`, and how many writes `behind`."""
|
|
759
|
+
status, _, raw = self._send("GET", f"{self._base}/_changes/consumers", None, self._auth(), twice=True)
|
|
760
|
+
return _answer(status, raw)
|
|
761
|
+
|
|
762
|
+
def forget_consumer(self, name: str) -> None:
|
|
763
|
+
status, _, raw = self._send("DELETE", f"{self._base}/_changes/consumers/{_seg(name)}", None, self._auth())
|
|
764
|
+
_answer(status, raw)
|
|
765
|
+
|
|
766
|
+
def follow(self, consumer: str, *, limit: int = 1000, wait: float = 10.0) -> Iterator[list]:
|
|
767
|
+
"""Each batch of writes for `consumer`, made if it is new, committed
|
|
768
|
+
once the loop asks for the next: a batch the loop did not finish is
|
|
769
|
+
read again, so each write comes at least once. Started again, it
|
|
770
|
+
goes on from the last batch committed."""
|
|
771
|
+
self.consumer(consumer)
|
|
772
|
+
while True:
|
|
773
|
+
got = self.changes(consumer=consumer, limit=limit, wait=wait)
|
|
774
|
+
# Nothing read, nothing committed: a commit is a write, which
|
|
775
|
+
# the stream passes over, and committing past it every time
|
|
776
|
+
# would write once a wait.
|
|
777
|
+
if got.writes:
|
|
778
|
+
yield got.writes
|
|
779
|
+
self.move_consumer(consumer, got.next)
|
|
780
|
+
|
|
781
|
+
def _auth(self) -> dict:
|
|
782
|
+
return {"Authorization": f"Bearer {self.token}"} if self.token else {}
|
|
783
|
+
|
|
784
|
+
def _post(
|
|
785
|
+
self,
|
|
786
|
+
path: str,
|
|
787
|
+
body: bytes,
|
|
788
|
+
content_type: str,
|
|
789
|
+
after: int | None = None,
|
|
790
|
+
accept: int = 0,
|
|
791
|
+
key: str | None = None,
|
|
792
|
+
wait: float | None = None,
|
|
793
|
+
read: bool = False,
|
|
794
|
+
) -> Any:
|
|
795
|
+
headers = {"Content-Type": content_type}
|
|
796
|
+
if self.token:
|
|
797
|
+
headers["Authorization"] = f"Bearer {self.token}"
|
|
798
|
+
near = read and self.reads == "nearest"
|
|
799
|
+
if near:
|
|
800
|
+
sent = self._shared.at
|
|
801
|
+
_near(headers, sent, after)
|
|
802
|
+
elif after is not None:
|
|
803
|
+
headers["Fennec-After"] = str(after)
|
|
804
|
+
key = key or self.idempotency_key
|
|
805
|
+
if key:
|
|
806
|
+
headers["Idempotency-Key"] = key
|
|
807
|
+
if wait:
|
|
808
|
+
headers["Fennec-Wait"] = _wait_ms(wait)
|
|
809
|
+
timeout = _more(self.timeout, wait)
|
|
810
|
+
# What may run twice, sent again after a reset too: a read the
|
|
811
|
+
# builder made (`_BUILT`), one sent near, which every server runs
|
|
812
|
+
# read-only (`Fennec-Read`), and a write under a key. The caller's
|
|
813
|
+
# `read=True` is a claim nothing checks at home: a call it names
|
|
814
|
+
# may have been made to write since.
|
|
815
|
+
twice = read is _BUILT or bool(key)
|
|
816
|
+
status, got, raw = self._send("POST", self._base + path, body, headers, timeout, twice or near)
|
|
817
|
+
# 504: no copy the router reached holds that write in the wait --
|
|
818
|
+
# the write lost in a failover of home, which says where it stands
|
|
819
|
+
# now: the client's position from here on. Asked home with no mark,
|
|
820
|
+
# the read is answered with what home holds.
|
|
821
|
+
if near and status == 504 and "Fennec-After" in headers:
|
|
822
|
+
# A change the caller named is not the client's position.
|
|
823
|
+
if after is None and sent is not None:
|
|
824
|
+
self._shared.told(raw, sent)
|
|
825
|
+
for h in ("Fennec-Read", "Fennec-After", "Fennec-History"):
|
|
826
|
+
headers.pop(h, None)
|
|
827
|
+
near = False
|
|
828
|
+
status, got, raw = self._send("POST", self._base + path, body, headers, timeout, twice)
|
|
829
|
+
if 200 <= status < 300:
|
|
830
|
+
# A replayed batch's answer carries no `Fennec-Seq`: `seq` stays
|
|
831
|
+
# where the last write that sent one left it. A read near's
|
|
832
|
+
# answer may be a replica's: it moves nothing.
|
|
833
|
+
seq = got.get("Fennec-Seq")
|
|
834
|
+
if seq is not None and not near:
|
|
835
|
+
self._shared.answered(int(seq), got.get("Fennec-History"))
|
|
836
|
+
self._shared.replayed = got.get("Idempotent-Replayed") == "true"
|
|
837
|
+
# An answer that is no error: a schema's refusals (409).
|
|
838
|
+
if status == accept:
|
|
839
|
+
return json.loads(raw)
|
|
840
|
+
return _answer(status, raw)
|
|
841
|
+
|
|
842
|
+
def _send(
|
|
843
|
+
self,
|
|
844
|
+
method: str,
|
|
845
|
+
path: str,
|
|
846
|
+
body: bytes | None,
|
|
847
|
+
headers: dict,
|
|
848
|
+
timeout: float | None = None,
|
|
849
|
+
twice: bool = False,
|
|
850
|
+
) -> tuple:
|
|
851
|
+
"""A request, its status, headers and body, sent again by the rule
|
|
852
|
+
the router keeps: only what cannot have run, or may run twice. An
|
|
853
|
+
answer naming its wait -- a 503, a quota's 429 -- ran nothing, unless
|
|
854
|
+
it says statements of a batch were made (`_wait_of`), nor did a
|
|
855
|
+
request whose connection was refused or that was not handed to the
|
|
856
|
+
system whole: any request goes again, after that wait and never
|
|
857
|
+
sooner. `twice` -- a read the builder made or sent near, a write
|
|
858
|
+
under a key -- goes again after a reset or a dropped connection too,
|
|
859
|
+
which a write may have landed before. Nothing goes again after the
|
|
860
|
+
client's own timeout, past which the server may still run it, nor a
|
|
861
|
+
refusal naming no wait; past the `Retry`'s tries or its waits, the
|
|
862
|
+
last answer or error is the caller's."""
|
|
863
|
+
tries, waited = 0, 0.0
|
|
864
|
+
while True:
|
|
865
|
+
tries += 1
|
|
866
|
+
error = None
|
|
867
|
+
try:
|
|
868
|
+
status, got, raw = self._once(method, path, body, headers, timeout)
|
|
869
|
+
# Headers asked of a 429 or a 503 alone: an `HTTPMessage`'s
|
|
870
|
+
# `get` walks every header.
|
|
871
|
+
wait = _wait_of(status, raw, got.get("Retry-After")) if status in (429, 503) else None
|
|
872
|
+
if wait is None:
|
|
873
|
+
return status, got, raw
|
|
874
|
+
except TimeoutError:
|
|
875
|
+
# The client's own: a `compact`, a long read may still be
|
|
876
|
+
# running, and three tries ran a read three times.
|
|
877
|
+
raise
|
|
878
|
+
except _Unsent as e:
|
|
879
|
+
error, wait = e.__cause__, 0.0
|
|
880
|
+
except (OSError, http.client.HTTPException) as e:
|
|
881
|
+
if not (twice or isinstance(e, ConnectionRefusedError)):
|
|
882
|
+
raise
|
|
883
|
+
error, wait = e, 0.0
|
|
884
|
+
pause = _pause(self.retry, tries, waited, wait)
|
|
885
|
+
if pause is None:
|
|
886
|
+
if error is not None:
|
|
887
|
+
raise error
|
|
888
|
+
return status, got, raw
|
|
889
|
+
waited += pause
|
|
890
|
+
time.sleep(pause)
|
|
891
|
+
|
|
892
|
+
def _once(
|
|
893
|
+
self, method: str, path: str, body: bytes | None, headers: dict, timeout: float | None
|
|
894
|
+
) -> tuple:
|
|
895
|
+
"""One try over this thread's connection: the connect and the
|
|
896
|
+
answer's head within `timeout` -- each wait given what is left of it
|
|
897
|
+
-- and each piece of its body within `self.timeout` of the last, so
|
|
898
|
+
a long answer is read for as long as it comes and one that stops is
|
|
899
|
+
given up. Bounded whole, a long read failed however steadily it
|
|
900
|
+
came. A kept connection the server closed while it was idle is found
|
|
901
|
+
so before the request goes -- its socket reads as ready, the end of
|
|
902
|
+
the stream -- and a new one is opened in its place: a try on it
|
|
903
|
+
would fail as though the request may have run."""
|
|
904
|
+
limit = self.timeout if timeout is None else timeout
|
|
905
|
+
end = None if limit is None else time.monotonic() + limit
|
|
906
|
+
|
|
907
|
+
def left() -> float | None:
|
|
908
|
+
return None if end is None else end - time.monotonic()
|
|
909
|
+
|
|
910
|
+
conn = getattr(self._local, "conn", None)
|
|
911
|
+
if conn is not None and _closed(conn):
|
|
912
|
+
conn.close()
|
|
913
|
+
conn = None
|
|
914
|
+
if conn is None:
|
|
915
|
+
conn = self._connect()
|
|
916
|
+
try:
|
|
917
|
+
if conn.sock is None:
|
|
918
|
+
conn.timeout = limit
|
|
919
|
+
conn.connect()
|
|
920
|
+
sock = conn.sock
|
|
921
|
+
_bound(sock, left())
|
|
922
|
+
try:
|
|
923
|
+
conn.request(method, path, body, headers)
|
|
924
|
+
except TimeoutError:
|
|
925
|
+
# A server that stopped reading: sent again, it would wait as
|
|
926
|
+
# long ([`TimeoutError`] in `_send`).
|
|
927
|
+
raise
|
|
928
|
+
except OSError as e:
|
|
929
|
+
# Not handed to the system whole, as the router judges it:
|
|
930
|
+
# the server read no request to run.
|
|
931
|
+
raise _Unsent(e) from e
|
|
932
|
+
# A small request goes at once, the bound before it as good for
|
|
933
|
+
# the head; a large one may take its time going.
|
|
934
|
+
if body is not None and len(body) > 65536:
|
|
935
|
+
_bound(sock, left())
|
|
936
|
+
resp = conn.getresponse()
|
|
937
|
+
# `read` gives each wait for a piece of the body the socket's
|
|
938
|
+
# timeout: the timeout between pieces.
|
|
939
|
+
_bound(sock, self.timeout)
|
|
940
|
+
raw = resp.read()
|
|
941
|
+
except BaseException:
|
|
942
|
+
conn.close()
|
|
943
|
+
self._local.conn = None
|
|
944
|
+
raise
|
|
945
|
+
if resp.will_close:
|
|
946
|
+
conn.close()
|
|
947
|
+
self._local.conn = None
|
|
948
|
+
else:
|
|
949
|
+
self._local.conn = conn
|
|
950
|
+
return resp.status, resp.msg, raw
|
|
951
|
+
|
|
952
|
+
def _connect(self) -> http.client.HTTPConnection:
|
|
953
|
+
if self._tls:
|
|
954
|
+
return http.client.HTTPSConnection(
|
|
955
|
+
self._host, self._port, timeout=self.timeout, context=ssl.create_default_context()
|
|
956
|
+
)
|
|
957
|
+
return http.client.HTTPConnection(self._host, self._port, timeout=self.timeout)
|
|
958
|
+
|
|
959
|
+
def close(self) -> None:
|
|
960
|
+
"""Closes this thread's connection; the next request opens another."""
|
|
961
|
+
conn = getattr(self._local, "conn", None)
|
|
962
|
+
if conn is not None:
|
|
963
|
+
conn.close()
|
|
964
|
+
self._local.conn = None
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
class _Unsent(Exception):
|
|
968
|
+
"""A request not handed to the system whole: its cause, the error that
|
|
969
|
+
stopped it, is what the caller meets once no try is left."""
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _more(timeout: float | None, wait: float | None) -> float | None:
|
|
973
|
+
"""A request's timeout, and the `wait` the server may hold it on top."""
|
|
974
|
+
return None if timeout is None else timeout + (wait or 0)
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def _wait_ms(wait: float) -> str:
|
|
978
|
+
"""`Fennec-Wait` for `wait` seconds: milliseconds, as the header is."""
|
|
979
|
+
if isinstance(wait, bool) or not isinstance(wait, (int, float)) or wait < 0:
|
|
980
|
+
raise ValueError(f"wait is seconds, a number from 0 (got {wait!r})")
|
|
981
|
+
return str(int(wait * 1000))
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _seg(name: str) -> str:
|
|
985
|
+
return urllib.parse.quote(name, safe="")
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _faceted(answer: Any) -> Any:
|
|
989
|
+
"""A `/query` answer as it came, but for a query that asked facets:
|
|
990
|
+
the server sends `{"rows": [...], "facets": {...}}` for it rather than
|
|
991
|
+
the bare array, and that becomes `Rows` -- a list of the rows, as every
|
|
992
|
+
other read answers, with the counts as its `facets`. A `/batch` answer
|
|
993
|
+
is left as it came, each read's result `{"rows": ..., "facets": ...}`."""
|
|
994
|
+
if isinstance(answer, dict) and "facets" in answer and isinstance(answer.get("rows"), list):
|
|
995
|
+
return _rows(answer)
|
|
996
|
+
return answer
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _closed(conn: http.client.HTTPConnection) -> bool:
|
|
1000
|
+
"""Whether a kept connection can no longer take a request: the server
|
|
1001
|
+
closed it (its socket reads as ready, at the end of the stream) or sent
|
|
1002
|
+
what no request asked for. Between requests nothing is due on it."""
|
|
1003
|
+
sock = conn.sock
|
|
1004
|
+
if sock is None:
|
|
1005
|
+
return True
|
|
1006
|
+
try:
|
|
1007
|
+
ready, _, _ = select.select([sock], [], [], 0)
|
|
1008
|
+
except (OSError, ValueError):
|
|
1009
|
+
return True
|
|
1010
|
+
return bool(ready)
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
def _answer(status: int, raw: bytes) -> Any:
|
|
1014
|
+
"""A response's JSON, or the error it says, as `Client` raises it."""
|
|
1015
|
+
if 200 <= status < 300:
|
|
1016
|
+
return json.loads(raw) if raw else None
|
|
1017
|
+
at = completed = quota = retry_after = outcome = seq = history = reason = binding = None
|
|
1018
|
+
try:
|
|
1019
|
+
body = json.loads(raw)
|
|
1020
|
+
message = body.get("error", raw.decode(errors="replace"))
|
|
1021
|
+
at, completed = body.get("at"), body.get("completed")
|
|
1022
|
+
# What a `require n else '<reason>'` that refused said, a batch's
|
|
1023
|
+
# stop's beside its `at`.
|
|
1024
|
+
reason = body.get("reason")
|
|
1025
|
+
# The statement bound to the write that refused it, by name.
|
|
1026
|
+
binding = body.get("binding")
|
|
1027
|
+
# A quota's refusal says which, and the wait its `Retry-After`
|
|
1028
|
+
# says -- a quota's 429, a move's 503 -- in the body too: what is
|
|
1029
|
+
# read here has no headers. To the millisecond where it says it: a
|
|
1030
|
+
# bucket of 600 rows a second refills a row in 1.7 ms.
|
|
1031
|
+
quota, retry_after = body.get("quota"), body.get("retry_after")
|
|
1032
|
+
ms = body.get("retry_after_ms")
|
|
1033
|
+
if isinstance(ms, (int, float)) and ms > 0:
|
|
1034
|
+
retry_after = ms / 1000
|
|
1035
|
+
# A write a tenant's node answered after its lease lapsed: neither
|
|
1036
|
+
# made nor refused, its position for a read once the tenant settles.
|
|
1037
|
+
if body.get("outcome") == "unknown":
|
|
1038
|
+
outcome, seq, history = "unknown", body.get("seq"), body.get("history")
|
|
1039
|
+
except (ValueError, AttributeError):
|
|
1040
|
+
message = raw.decode(errors="replace")
|
|
1041
|
+
raise FennecError(
|
|
1042
|
+
message,
|
|
1043
|
+
status,
|
|
1044
|
+
at=at,
|
|
1045
|
+
completed=completed,
|
|
1046
|
+
quota=quota,
|
|
1047
|
+
retry_after=retry_after,
|
|
1048
|
+
outcome=outcome,
|
|
1049
|
+
seq=seq,
|
|
1050
|
+
history=history,
|
|
1051
|
+
reason=reason,
|
|
1052
|
+
binding=binding,
|
|
1053
|
+
)
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
class AsyncClient:
|
|
1057
|
+
"""fennec-server's HTTP endpoint over asyncio: `Client`'s calls, awaited.
|
|
1058
|
+
|
|
1059
|
+
One connection, kept alive and opened again when the server closed it;
|
|
1060
|
+
calls made at once on one client go one after another over it, as they
|
|
1061
|
+
would over one `Client` -- a client each for requests side by side.
|
|
1062
|
+
`https://` for a server behind a TLS terminator. The standard library
|
|
1063
|
+
alone: asyncio's streams and a request written out by hand, the
|
|
1064
|
+
endpoint's answers being a status, a length and JSON. `timeout` and
|
|
1065
|
+
`retry` are `Client`'s.
|
|
1066
|
+
"""
|
|
1067
|
+
|
|
1068
|
+
def __init__(
|
|
1069
|
+
self,
|
|
1070
|
+
url: str = "http://127.0.0.1:8080",
|
|
1071
|
+
token: str | None = None,
|
|
1072
|
+
timeout: float | None = 30.0,
|
|
1073
|
+
reads: str = "home",
|
|
1074
|
+
retry: Retry | bool = True,
|
|
1075
|
+
):
|
|
1076
|
+
parts = urllib.parse.urlsplit(url)
|
|
1077
|
+
if parts.scheme not in ("http", "https"):
|
|
1078
|
+
raise ValueError(f"not an http(s) URL: {url!r}")
|
|
1079
|
+
self.url = url.rstrip("/")
|
|
1080
|
+
self.token = token
|
|
1081
|
+
self.timeout = timeout
|
|
1082
|
+
self.retry = _retry(retry)
|
|
1083
|
+
self.reads = _reads(reads)
|
|
1084
|
+
self._host = parts.hostname or "127.0.0.1"
|
|
1085
|
+
self._port = parts.port or (443 if parts.scheme == "https" else 80)
|
|
1086
|
+
self._tls = ssl.create_default_context() if parts.scheme == "https" else None
|
|
1087
|
+
self._base = parts.path.rstrip("/")
|
|
1088
|
+
self.idempotency_key: str | None = None
|
|
1089
|
+
# The connection, its lock, `seq` and `replayed`, shared with the
|
|
1090
|
+
# copies `with_idempotency_key` makes: one connection still.
|
|
1091
|
+
self._s = _AsyncShared()
|
|
1092
|
+
|
|
1093
|
+
@property
|
|
1094
|
+
def seq(self) -> int | None:
|
|
1095
|
+
"""The change the last write left the database at, as `Client.seq`."""
|
|
1096
|
+
return self._s.seq
|
|
1097
|
+
|
|
1098
|
+
@property
|
|
1099
|
+
def replayed(self) -> bool:
|
|
1100
|
+
"""Whether the last write's answer was replayed, as `Client.replayed`."""
|
|
1101
|
+
return self._s.replayed
|
|
1102
|
+
|
|
1103
|
+
def with_idempotency_key(self, key: str) -> "AsyncClient":
|
|
1104
|
+
"""A copy whose writes carry `key`, as `Client.with_idempotency_key`,
|
|
1105
|
+
over this client's connection."""
|
|
1106
|
+
if not isinstance(key, str) or not key:
|
|
1107
|
+
raise ValueError("an idempotency key is a text, not empty")
|
|
1108
|
+
copy = object.__new__(AsyncClient)
|
|
1109
|
+
copy.__dict__.update(self.__dict__)
|
|
1110
|
+
copy.idempotency_key = key
|
|
1111
|
+
return copy
|
|
1112
|
+
|
|
1113
|
+
def collection(self, name: str) -> AsyncQuery:
|
|
1114
|
+
"""The query builder, as `Client.collection`, its endpoints awaited."""
|
|
1115
|
+
return AsyncQuery(name, self)
|
|
1116
|
+
|
|
1117
|
+
async def call(
|
|
1118
|
+
self,
|
|
1119
|
+
name: str,
|
|
1120
|
+
args: Mapping[str, Any] | None = None,
|
|
1121
|
+
*,
|
|
1122
|
+
claims: Mapping[str, Any] | None = None,
|
|
1123
|
+
idempotency_key: str | None = None,
|
|
1124
|
+
wait: float | None = None,
|
|
1125
|
+
read: bool = False,
|
|
1126
|
+
) -> Any:
|
|
1127
|
+
"""A named statement's call, as `Client.call`, awaited; a held one is
|
|
1128
|
+
cancelled as any task is, its connection closed with it."""
|
|
1129
|
+
text, params = Call(name, args, claims=claims).to_call()
|
|
1130
|
+
return await self.query(text, params, idempotency_key=idempotency_key, wait=wait, read=read)
|
|
1131
|
+
|
|
1132
|
+
async def statements(self) -> list[NamedStatement]:
|
|
1133
|
+
"""The named statements the database holds, as `Client.statements`."""
|
|
1134
|
+
return [_named(r) for r in await self.query("statements")]
|
|
1135
|
+
|
|
1136
|
+
async def query(
|
|
1137
|
+
self,
|
|
1138
|
+
fennecql: str,
|
|
1139
|
+
params: Sequence[Any] | None = None,
|
|
1140
|
+
*,
|
|
1141
|
+
idempotency_key: str | None = None,
|
|
1142
|
+
wait: float | None = None,
|
|
1143
|
+
read: bool = False,
|
|
1144
|
+
) -> Any:
|
|
1145
|
+
"""One FennecQL statement, as `Client.query`; a held one is
|
|
1146
|
+
cancelled as any task is, its connection closed with it."""
|
|
1147
|
+
body = {"query": fennecql, "params": list(params or [])}
|
|
1148
|
+
return _faceted(
|
|
1149
|
+
await self._post(
|
|
1150
|
+
"/query", json.dumps(body).encode(), "application/json", idempotency_key, wait, read
|
|
1151
|
+
)
|
|
1152
|
+
)
|
|
1153
|
+
|
|
1154
|
+
async def batch(
|
|
1155
|
+
self,
|
|
1156
|
+
statements: Iterable[tuple[str, Sequence[Any]]],
|
|
1157
|
+
*,
|
|
1158
|
+
idempotency_key: str | None = None,
|
|
1159
|
+
wait: float | None = None,
|
|
1160
|
+
read: bool = False,
|
|
1161
|
+
) -> Any:
|
|
1162
|
+
"""Statements under one write lock, as one block, as `Client.batch`."""
|
|
1163
|
+
lines = [json.dumps({"query": q, "params": list(p)}) for q, p in statements]
|
|
1164
|
+
return await self._post(
|
|
1165
|
+
"/batch", "\n".join(lines).encode(), "application/x-ndjson", idempotency_key, wait, read
|
|
1166
|
+
)
|
|
1167
|
+
|
|
1168
|
+
async def close(self) -> None:
|
|
1169
|
+
if self._s.conn:
|
|
1170
|
+
_, writer = self._s.conn
|
|
1171
|
+
self._s.conn = None
|
|
1172
|
+
writer.close()
|
|
1173
|
+
try:
|
|
1174
|
+
await writer.wait_closed()
|
|
1175
|
+
except OSError:
|
|
1176
|
+
pass
|
|
1177
|
+
|
|
1178
|
+
async def __aenter__(self) -> "AsyncClient":
|
|
1179
|
+
return self
|
|
1180
|
+
|
|
1181
|
+
async def __aexit__(self, *exc: Any) -> None:
|
|
1182
|
+
await self.close()
|
|
1183
|
+
|
|
1184
|
+
async def _post(
|
|
1185
|
+
self,
|
|
1186
|
+
path: str,
|
|
1187
|
+
body: bytes,
|
|
1188
|
+
content_type: str,
|
|
1189
|
+
key: str | None = None,
|
|
1190
|
+
wait: float | None = None,
|
|
1191
|
+
read: bool = False,
|
|
1192
|
+
) -> Any:
|
|
1193
|
+
key = key or self.idempotency_key
|
|
1194
|
+
held = _wait_ms(wait) if wait else None
|
|
1195
|
+
# As `Client._post`'s: what may run twice -- a read the builder made,
|
|
1196
|
+
# one sent near, a write under a key -- goes again after a reset too.
|
|
1197
|
+
twice = read is _BUILT or bool(key)
|
|
1198
|
+
limit = _more(self.timeout, wait)
|
|
1199
|
+
async with self._s.lock:
|
|
1200
|
+
# A read near, as `Client`'s: marked, after this client's
|
|
1201
|
+
# furthest position, and asked home on a 504, taking home's.
|
|
1202
|
+
near: dict = {}
|
|
1203
|
+
if read and self.reads == "nearest":
|
|
1204
|
+
sent = self._s.at
|
|
1205
|
+
_near(near, sent, None)
|
|
1206
|
+
tries, waited = 0, 0.0
|
|
1207
|
+
while True:
|
|
1208
|
+
tries += 1
|
|
1209
|
+
error = None
|
|
1210
|
+
try:
|
|
1211
|
+
status, headers, raw = await self._try(path, body, content_type, key, held, near, limit)
|
|
1212
|
+
if raw is None:
|
|
1213
|
+
raw = await self._body(headers)
|
|
1214
|
+
if status == 504 and "Fennec-After" in near:
|
|
1215
|
+
self._s.told(raw, sent)
|
|
1216
|
+
near = {}
|
|
1217
|
+
status, headers, raw = await self._try(path, body, content_type, key, held, near, limit)
|
|
1218
|
+
if raw is None:
|
|
1219
|
+
raw = await self._body(headers)
|
|
1220
|
+
wait_s = _wait_of(status, raw, headers.get("retry-after"))
|
|
1221
|
+
if wait_s is None:
|
|
1222
|
+
if 200 <= status < 300:
|
|
1223
|
+
if "fennec-seq" in headers and not near:
|
|
1224
|
+
self._s.answered(int(headers["fennec-seq"]), headers.get("fennec-history"))
|
|
1225
|
+
self._s.replayed = headers.get("idempotent-replayed") == "true"
|
|
1226
|
+
return _answer(status, raw)
|
|
1227
|
+
except (TimeoutError, asyncio.TimeoutError):
|
|
1228
|
+
# The client's own, as `Client._send`'s: never again.
|
|
1229
|
+
await self.close()
|
|
1230
|
+
raise
|
|
1231
|
+
except (OSError, asyncio.IncompleteReadError) as e:
|
|
1232
|
+
await self.close()
|
|
1233
|
+
# Refused, at every address of the host's: nothing went,
|
|
1234
|
+
# and any request goes again.
|
|
1235
|
+
if not (twice or near or _refused(e)):
|
|
1236
|
+
raise
|
|
1237
|
+
error, wait_s = e, 0.0
|
|
1238
|
+
except asyncio.CancelledError:
|
|
1239
|
+
# Cancelled under way -- a held claim given up: its
|
|
1240
|
+
# answer, still to come, must not be the next request's.
|
|
1241
|
+
self._drop()
|
|
1242
|
+
raise
|
|
1243
|
+
pause = _pause(self.retry, tries, waited, wait_s)
|
|
1244
|
+
if pause is None:
|
|
1245
|
+
if error is not None:
|
|
1246
|
+
raise error
|
|
1247
|
+
return _answer(status, raw)
|
|
1248
|
+
waited += pause
|
|
1249
|
+
await asyncio.sleep(pause)
|
|
1250
|
+
|
|
1251
|
+
def _try(
|
|
1252
|
+
self,
|
|
1253
|
+
path: str,
|
|
1254
|
+
body: bytes,
|
|
1255
|
+
content_type: str,
|
|
1256
|
+
key: str | None,
|
|
1257
|
+
held: str | None,
|
|
1258
|
+
extra: dict,
|
|
1259
|
+
limit: float | None,
|
|
1260
|
+
) -> Any:
|
|
1261
|
+
"""One try, to be awaited: the connect, the answer's head and a body
|
|
1262
|
+
in hand with it within `limit` -- a body still to come is left to
|
|
1263
|
+
`_body`, None in its place. A kept connection the server closed
|
|
1264
|
+
while it was idle is let go of before the request goes, as `Client`
|
|
1265
|
+
finds one: a try on it would fail as one that may have run. Not a
|
|
1266
|
+
coroutine of its own: one more a request cost 0.4 us."""
|
|
1267
|
+
if self._s.conn is not None and _gone(*self._s.conn):
|
|
1268
|
+
self._drop()
|
|
1269
|
+
return asyncio.wait_for(self._exchange(path, body, content_type, key, held, extra), limit)
|
|
1270
|
+
|
|
1271
|
+
def _drop(self) -> None:
|
|
1272
|
+
if self._s.conn:
|
|
1273
|
+
_, writer = self._s.conn
|
|
1274
|
+
self._s.conn = None
|
|
1275
|
+
writer.close()
|
|
1276
|
+
|
|
1277
|
+
async def _exchange(
|
|
1278
|
+
self,
|
|
1279
|
+
path: str,
|
|
1280
|
+
body: bytes,
|
|
1281
|
+
content_type: str,
|
|
1282
|
+
key: str | None,
|
|
1283
|
+
held: str | None,
|
|
1284
|
+
extra: dict,
|
|
1285
|
+
) -> tuple[int, dict, bytes | None]:
|
|
1286
|
+
"""The request sent and the answer's head read (`_head`), and its
|
|
1287
|
+
body where the loop holds it whole already -- a head and a body the
|
|
1288
|
+
server wrote at once, as it writes every answer: nothing to wait
|
|
1289
|
+
for. Any other body is `_body`'s, read past the head's bound a piece
|
|
1290
|
+
at a time: one more coroutine and a timer the loop holds cost every
|
|
1291
|
+
request 3.5 us, 6%."""
|
|
1292
|
+
status, headers = await self._head(path, body, content_type, key, held, extra)
|
|
1293
|
+
n = headers.get("content-length")
|
|
1294
|
+
reader = self._s.conn[0]
|
|
1295
|
+
# CPython's buffer, and uvloop's, which runs asyncio's streams: a
|
|
1296
|
+
# reader without one has its body read by `_body`.
|
|
1297
|
+
if n is not None and len(getattr(reader, "_buffer", b"")) >= int(n):
|
|
1298
|
+
raw = await reader.readexactly(int(n))
|
|
1299
|
+
if headers.get("connection", "").lower() == "close":
|
|
1300
|
+
await self.close()
|
|
1301
|
+
return status, headers, raw
|
|
1302
|
+
return status, headers, None
|
|
1303
|
+
|
|
1304
|
+
async def _body(self, headers: dict) -> bytes:
|
|
1305
|
+
"""The body of the answer whose `headers` were read, each piece
|
|
1306
|
+
within the timeout of the last (`_Idle`): a long answer is read for
|
|
1307
|
+
as long as it comes, one that stops is given up, its connection
|
|
1308
|
+
with it."""
|
|
1309
|
+
reader, writer = self._s.conn
|
|
1310
|
+
idle = _Idle(writer, self.timeout)
|
|
1311
|
+
try:
|
|
1312
|
+
if "content-length" in headers:
|
|
1313
|
+
raw = await _exactly(reader, int(headers["content-length"]), idle)
|
|
1314
|
+
elif headers.get("transfer-encoding", "").lower() == "chunked":
|
|
1315
|
+
parts = []
|
|
1316
|
+
while True:
|
|
1317
|
+
idle.tick()
|
|
1318
|
+
size = int((await reader.readuntil(b"\r\n")).split(b";")[0], 16)
|
|
1319
|
+
if not size:
|
|
1320
|
+
break
|
|
1321
|
+
parts.append(await _exactly(reader, size + 2, idle))
|
|
1322
|
+
idle.tick()
|
|
1323
|
+
await reader.readuntil(b"\r\n")
|
|
1324
|
+
raw = b"".join(p[:-2] for p in parts)
|
|
1325
|
+
else:
|
|
1326
|
+
parts = []
|
|
1327
|
+
while True:
|
|
1328
|
+
idle.tick()
|
|
1329
|
+
part = await reader.read(1 << 16)
|
|
1330
|
+
if not part:
|
|
1331
|
+
break
|
|
1332
|
+
parts.append(part)
|
|
1333
|
+
raw = b"".join(parts)
|
|
1334
|
+
self._s.conn = None
|
|
1335
|
+
except (OSError, asyncio.IncompleteReadError):
|
|
1336
|
+
if idle.late:
|
|
1337
|
+
raise asyncio.TimeoutError(f"no piece of the answer within {self.timeout} s") from None
|
|
1338
|
+
raise
|
|
1339
|
+
finally:
|
|
1340
|
+
idle.stop()
|
|
1341
|
+
if headers.get("connection", "").lower() == "close" and self._s.conn:
|
|
1342
|
+
await self.close()
|
|
1343
|
+
return raw
|
|
1344
|
+
|
|
1345
|
+
async def _head(
|
|
1346
|
+
self,
|
|
1347
|
+
path: str,
|
|
1348
|
+
body: bytes,
|
|
1349
|
+
content_type: str,
|
|
1350
|
+
key: str | None,
|
|
1351
|
+
held: str | None,
|
|
1352
|
+
extra: dict,
|
|
1353
|
+
) -> tuple[int, dict]:
|
|
1354
|
+
"""The request sent, over a connection opened where none is kept,
|
|
1355
|
+
and the answer's status and headers."""
|
|
1356
|
+
if self._s.conn is None:
|
|
1357
|
+
self._s.conn = await asyncio.open_connection(
|
|
1358
|
+
self._host, self._port, ssl=self._tls, server_hostname=self._host if self._tls else None
|
|
1359
|
+
)
|
|
1360
|
+
reader, writer = self._s.conn
|
|
1361
|
+
head = [
|
|
1362
|
+
f"POST {self._base}{path} HTTP/1.1",
|
|
1363
|
+
f"Host: {self._host}:{self._port}",
|
|
1364
|
+
f"Content-Type: {content_type}",
|
|
1365
|
+
f"Content-Length: {len(body)}",
|
|
1366
|
+
"Connection: keep-alive",
|
|
1367
|
+
]
|
|
1368
|
+
if self.token:
|
|
1369
|
+
head.append(f"Authorization: Bearer {self.token}")
|
|
1370
|
+
if key:
|
|
1371
|
+
head.append(f"Idempotency-Key: {key}")
|
|
1372
|
+
if held:
|
|
1373
|
+
head.append(f"Fennec-Wait: {held}")
|
|
1374
|
+
for name, value in extra.items():
|
|
1375
|
+
head.append(f"{name}: {value}")
|
|
1376
|
+
writer.write(("\r\n".join(head) + "\r\n\r\n").encode() + body)
|
|
1377
|
+
await writer.drain()
|
|
1378
|
+
|
|
1379
|
+
status_line = await reader.readuntil(b"\r\n")
|
|
1380
|
+
status = int(status_line.split(b" ", 2)[1])
|
|
1381
|
+
headers: dict[str, str] = {}
|
|
1382
|
+
while (line := await reader.readuntil(b"\r\n")) != b"\r\n":
|
|
1383
|
+
name, _, value = line.decode("latin-1").partition(":")
|
|
1384
|
+
headers[name.strip().lower()] = value.strip()
|
|
1385
|
+
return status, headers
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
class _Idle:
|
|
1389
|
+
"""The clock of an answer's body: `secs` with nothing read, and its
|
|
1390
|
+
connection is let go of (`late`), the read under way meeting its end. A
|
|
1391
|
+
timer the loop holds, where `wait_for` made a task a wait before 3.12."""
|
|
1392
|
+
|
|
1393
|
+
__slots__ = ("secs", "writer", "loop", "handle", "late")
|
|
1394
|
+
|
|
1395
|
+
def __init__(self, writer: asyncio.StreamWriter, secs: float | None):
|
|
1396
|
+
self.secs, self.writer, self.handle, self.late = secs, writer, None, False
|
|
1397
|
+
self.loop = asyncio.get_running_loop()
|
|
1398
|
+
|
|
1399
|
+
def tick(self) -> None:
|
|
1400
|
+
"""Starts the clock again: a piece came, or one is to be read."""
|
|
1401
|
+
if self.handle is not None:
|
|
1402
|
+
self.handle.cancel()
|
|
1403
|
+
if self.secs is not None:
|
|
1404
|
+
self.handle = self.loop.call_later(self.secs, self._late)
|
|
1405
|
+
|
|
1406
|
+
def _late(self) -> None:
|
|
1407
|
+
self.late = True
|
|
1408
|
+
self.writer.transport.abort()
|
|
1409
|
+
|
|
1410
|
+
def stop(self) -> None:
|
|
1411
|
+
if self.handle is not None:
|
|
1412
|
+
self.handle.cancel()
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
async def _exactly(reader: asyncio.StreamReader, n: int, idle: _Idle) -> bytes:
|
|
1416
|
+
"""`n` bytes of `reader`, each wait for a piece within `idle`'s clock."""
|
|
1417
|
+
parts = []
|
|
1418
|
+
while n:
|
|
1419
|
+
idle.tick()
|
|
1420
|
+
part = await reader.read(n)
|
|
1421
|
+
if not part:
|
|
1422
|
+
raise asyncio.IncompleteReadError(b"".join(parts), None)
|
|
1423
|
+
parts.append(part)
|
|
1424
|
+
n -= len(part)
|
|
1425
|
+
return parts[0] if len(parts) == 1 else b"".join(parts)
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def _gone(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> bool:
|
|
1429
|
+
"""Whether a kept stream can no longer take a request: its end read, or
|
|
1430
|
+
an error, as the loop met them. The loop reads a server's close as it
|
|
1431
|
+
comes whenever it waits, as an idle one does; asking the socket as
|
|
1432
|
+
`_closed` does was 1 us a request more, for a loop blocked past the
|
|
1433
|
+
close alone -- whose write without a key then fails, never sent twice."""
|
|
1434
|
+
return reader.at_eof() or reader.exception() is not None or writer.is_closing()
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
class _AsyncShared(_Shared):
|
|
1438
|
+
"""An `AsyncClient`'s connection and what its copies share with it."""
|
|
1439
|
+
|
|
1440
|
+
__slots__ = ("conn", "lock")
|
|
1441
|
+
|
|
1442
|
+
def __init__(self) -> None:
|
|
1443
|
+
super().__init__()
|
|
1444
|
+
self.conn: tuple[asyncio.StreamReader, asyncio.StreamWriter] | None = None
|
|
1445
|
+
self.lock = asyncio.Lock()
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def placeholders(start: int, n: int) -> str:
|
|
1449
|
+
"""`$start, ..., $(start+n-1)`: a list literal's worth of parameters.
|
|
1450
|
+
FennecQL binds a parameter to a value, not to a list, so `in` takes one
|
|
1451
|
+
per element -- `in [$2, $3, $4]`."""
|
|
1452
|
+
return ", ".join(f"${i}" for i in range(start, start + n))
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
# Below FennecError, which the builder raises.
|
|
1456
|
+
from .builder import ( # noqa: E402
|
|
1457
|
+
AsyncQuery,
|
|
1458
|
+
Call,
|
|
1459
|
+
Computed,
|
|
1460
|
+
Cond,
|
|
1461
|
+
FacetCount,
|
|
1462
|
+
Param,
|
|
1463
|
+
Query,
|
|
1464
|
+
Rows,
|
|
1465
|
+
_BUILT,
|
|
1466
|
+
_rows,
|
|
1467
|
+
and_,
|
|
1468
|
+
bucket,
|
|
1469
|
+
call,
|
|
1470
|
+
collection,
|
|
1471
|
+
count_distinct,
|
|
1472
|
+
distance,
|
|
1473
|
+
expr,
|
|
1474
|
+
first,
|
|
1475
|
+
inc,
|
|
1476
|
+
last,
|
|
1477
|
+
not_,
|
|
1478
|
+
or_,
|
|
1479
|
+
param,
|
|
1480
|
+
raw,
|
|
1481
|
+
)
|