fennecdb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fennecdb/__init__.py ADDED
@@ -0,0 +1,1481 @@
1
+ """fennecdb over HTTP.
2
+
3
+ A client for fennec-server's HTTP endpoint (`fennec-server --http <address>`), and on
4
+ top of it vector stores for LangChain (`fennecdb.langchain`) and LlamaIndex
5
+ (`fennecdb.llama_index`).
6
+
7
+ from fennecdb import Client
8
+ db = Client("http://127.0.0.1:8080", token="...")
9
+ db.query("get articles near embed $1 limit 5", [[0.1, 0.2, 0.3]])
10
+
11
+ The client is the standard library alone, as the server is: a statement is
12
+ one `POST /query`, several are one `POST /batch` under one lock.
13
+
14
+ `collection` starts the query builder, which writes the statement for you
15
+ -- every value a parameter, every name checked -- and makes the same text
16
+ the JavaScript, Go and .NET builders make of the same chain:
17
+
18
+ from fennecdb import or_
19
+ rows = (db.collection("articles").select("title")
20
+ .where("year", ">=", 2024)
21
+ .where(or_({"lang": "tr"}, {"tags": {"has": "rust"}}))
22
+ .near("embed", [0.1, 0.2, 0.3]).limit(5).rows())
23
+
24
+ `match` takes `highlight` and `snippet` marks in the select list, and any
25
+ read `facet` counts, which come back beside the rows as `rows.facets`:
26
+
27
+ rows = (db.collection("products").select("title").highlight("title")
28
+ .match("title", "phone").facet("brand", top=10).limit(20).rows())
29
+ rows.facets["brand"] # [FacetCount(value="acme", count=12), ...]
30
+
31
+ A named statement (`create statement transfer(...) as ...`) is called by
32
+ name with its arguments, an object, as one parameter -- `call transfer $1`
33
+ over `/query`, so the errors, the key and `wait` are a statement's; the
34
+ server's token may say the claims it is made with (`as $2`), and `read=True`
35
+ marks a statement that only reads, to go near a read replica:
36
+
37
+ db.call("transfer", {"from": 1, "to": 2, "amount": 300}, idempotency_key=transfer_id)
38
+ db.call("history", read=True)
39
+ text, params = call("transfer", {"from": 1, "to": 2, "amount": 300}).to_call()
40
+
41
+ A retry after a timeout cannot tell whether the write ran; under an
42
+ `Idempotency-Key` it runs once, and the second answer is the first one
43
+ kept (`replayed`). One key per write -- the same key with another request
44
+ is refused (422):
45
+
46
+ db.batch([debit, credit], idempotency_key=transfer_id)
47
+ db.with_idempotency_key(order_id).collection("orders").insert(order)
48
+
49
+ A worker's claim that finds no job can wait for one at the server
50
+ (`Fennec-Wait`): with `wait` a `set` or `del` that writes nothing is held
51
+ until it can -- a job enqueued, a delayed one come due, a lease lapsed --
52
+ or for that many seconds, at most 30, rather than sent again every few:
53
+
54
+ jobs = db.collection("jobs").where(raw("run_at <= now()")).order("run_at").limit(10)
55
+ claimed = jobs.update(claim, returning=True, wait=30)
56
+
57
+ `AsyncClient` is the same over asyncio, for an event loop a blocking
58
+ request would stall:
59
+
60
+ async with AsyncClient("http://127.0.0.1:8080", token="...") as db:
61
+ rows = await db.query("get articles near embed $1 limit 5", [[0.1, 0.2, 0.3]])
62
+
63
+ A client far from a tenant's home reads near itself where the tenant has a
64
+ read replica there (`fennec-shard`'s read replicas): with `reads="nearest"`
65
+ the builder's reads, and a statement given `read=True`, are marked
66
+ (`Fennec-Read: replica`) for a router near a replica to answer, each sent
67
+ after the furthest position an answer from home named to this client
68
+ (`Fennec-After` and `Fennec-History`: the change, and the history it is a
69
+ change of) -- it reads nothing older than it wrote, nor a write a failover
70
+ replaced. Writes go home:
71
+
72
+ db = Client("https://us.example.com/t/acme", token="...", reads="nearest")
73
+
74
+ `Client.follow` reads every write on the server's disk (`GET /_changes`,
75
+ `fennec-server --cdc`) for a consumer whose cursor the server keeps, and
76
+ commits each batch once the loop comes back for the next:
77
+
78
+ for batch in db.follow("search-index"):
79
+ for change in batch:
80
+ index(change) # an exception: this batch is read again
81
+
82
+ `python -m fennecdb.relay` hands the same to another program or a webhook.
83
+ """
84
+
85
+ from __future__ import annotations
86
+
87
+ import asyncio
88
+ import errno
89
+ import http.client
90
+ import json
91
+ import random
92
+ import re
93
+ import select
94
+ import socket
95
+ import ssl
96
+ import threading
97
+ import time
98
+ import urllib.parse
99
+ from datetime import datetime
100
+ from typing import Any, Iterable, Iterator, Mapping, NamedTuple, Sequence
101
+
102
+ __all__ = [
103
+ "AsyncClient",
104
+ "AsyncQuery",
105
+ "Call",
106
+ "Changes",
107
+ "Client",
108
+ "Computed",
109
+ "Cond",
110
+ "FacetCount",
111
+ "FennecError",
112
+ "HeldSlot",
113
+ "LastRun",
114
+ "NamedStatement",
115
+ "Param",
116
+ "Query",
117
+ "Retry",
118
+ "Rows",
119
+ "SchemaError",
120
+ "and_",
121
+ "bucket",
122
+ "call",
123
+ "collection",
124
+ "count_distinct",
125
+ "distance",
126
+ "expr",
127
+ "first",
128
+ "inc",
129
+ "last",
130
+ "not_",
131
+ "or_",
132
+ "param",
133
+ "placeholders",
134
+ "raw",
135
+ ]
136
+
137
+
138
+ class FennecError(Exception):
139
+ """A statement the server refused: its message, and the HTTP status --
140
+ 0 for one the query builder refused before sending anything. A batch
141
+ that stopped also says which statement stopped it, from 0 (`at`: the
142
+ write whose `require` was not met, 412, or whose id was taken, 409),
143
+ and how many statements stayed applied (`completed`: 0, since a batch
144
+ lands whole, but for one holding a `compact`); both None otherwise.
145
+
146
+ A `require n else '<reason>'` that refused says its reason (`reason`,
147
+ the builders' `reason=`): the statement's own words, by which a caller
148
+ tells "insufficient funds" from "no such account" -- where `at` moves
149
+ with every statement added in front of it. None where the statement
150
+ gave none.
151
+
152
+ A statement bound to the write (`create statement ... after`) that
153
+ refused it is named (`binding`): a guard's `require` not met, a write
154
+ of its body refused. Told the server's token alone, a scoped one the
155
+ `reason` and no name; None otherwise.
156
+
157
+ A tenant's quota names itself (`quota`): "bytes" and "rows" answer 507,
158
+ and room is made by deleting; "streams", "held", "requests", "writes"
159
+ and "write_bytes" answer 429, with the seconds to wait before sending
160
+ again (`retry_after`) -- to the millisecond, as the server says it in
161
+ `retry_after_ms` -- as a tenant being moved or failed over answers its
162
+ writes 503, in whole seconds. None otherwise.
163
+
164
+ A 500 whose `outcome` is "unknown" is a write a tenant's node landed and
165
+ answered after its lease from the router lapsed: its tenant may have
166
+ been failed over without it. Send it again with the same idempotency
167
+ key, or read it back a lease and a tenth later; `seq` and `history` say
168
+ where it left the node's database, and a read sent after them once the
169
+ tenant settles is answered by a primary holding the write and refused
170
+ 504 by one without it. Not the client's position, which this answer
171
+ does not move. None otherwise."""
172
+
173
+ def __init__(
174
+ self,
175
+ message: str,
176
+ status: int,
177
+ *,
178
+ at: int | None = None,
179
+ completed: int | None = None,
180
+ quota: str | None = None,
181
+ retry_after: float | None = None,
182
+ outcome: str | None = None,
183
+ seq: int | None = None,
184
+ history: str | None = None,
185
+ reason: str | None = None,
186
+ binding: str | None = None,
187
+ ):
188
+ super().__init__(message)
189
+ self.status = status
190
+ self.at = at
191
+ self.completed = completed
192
+ self.reason = reason
193
+ self.binding = binding
194
+ self.quota = quota
195
+ self.retry_after = retry_after
196
+ self.outcome = outcome
197
+ self.seq = seq
198
+ self.history = history
199
+
200
+
201
+ class SchemaError(FennecError):
202
+ """The schema the code declares and the server's differ in what no
203
+ open applies: each difference and how to resolve it, as the engine
204
+ writes them, in `refusals`."""
205
+
206
+ def __init__(self, refusals: list, status: int):
207
+ self.refusals = refusals
208
+ lines = [f" - {r['message']}\n {r['fix']}" for r in refusals]
209
+ super().__init__("the database's schema differs from the code's:\n" + "\n".join(lines), status)
210
+
211
+
212
+ class Changes(NamedTuple):
213
+ """Writes `GET /_changes` handed over, a dict each, and `next`: the last
214
+ one's number, which is the `since` to read on from. `seq` is the last
215
+ write the database holds (`Fennec-Seq`): a `next` that has reached it has
216
+ every write there is."""
217
+
218
+ writes: list
219
+ next: int
220
+ seq: int = 0
221
+
222
+
223
+ class LastRun(NamedTuple):
224
+ """A scheduled statement's last run, landed or pending, as `statements`
225
+ lists it: the slot it ran for (what `slot()` read), `"ran"` or
226
+ `"refused"` -- the statement's own refusal, a `require` not met or a
227
+ broken statement, which marks the slot as run -- its status and the
228
+ `reason` its `require` gave, and the slots passed over before it."""
229
+
230
+ slot: datetime
231
+ outcome: str
232
+ status: int | None
233
+ reason: str | None
234
+ skipped: int
235
+
236
+
237
+ class HeldSlot(NamedTuple):
238
+ """A due slot the server's state holds back -- a tenant's quota (507), a
239
+ storage error -- tried again as that changes, and since when."""
240
+
241
+ slot: datetime
242
+ status: int
243
+ since: datetime
244
+
245
+
246
+ class NamedStatement(NamedTuple):
247
+ """A named statement as `statements` lists it to the server's token,
248
+ each column None where it says nothing of the statement: a called one
249
+ has no `after` and no clock. `after` is a write-bound statement's
250
+ collection and the writes that run it (`("orders", "set or del")`);
251
+ `schedule` a scheduled one's clause as its text writes it (`every 1h`,
252
+ `at '0 3 * * *' in 'Europe/Berlin'`), `next` the slot due next -- past
253
+ while one is overdue or held -- `last` its last run and `held` a slot
254
+ held back. `hook` is a webhook consumer's hook, whose events are sent
255
+ to `/_hooks/<hook>`, and `armed` whether this server -- on a tenant
256
+ node, the tenant -- holds its secret: `False` answers every event of
257
+ it 401. `version` is the history and change its definition landed
258
+ at; `broken` why the schema broke it, which it runs no more until it
259
+ reads again."""
260
+
261
+ name: str
262
+ version: tuple[int, int]
263
+ signature: str | None
264
+ writes: bool | None
265
+ comment: str | None
266
+ broken: str | None
267
+ after: tuple[str, str] | None
268
+ schedule: str | None
269
+ next: datetime | None
270
+ last: LastRun | None
271
+ held: HeldSlot | None
272
+ hook: str | None = None
273
+ armed: bool | None = None
274
+
275
+
276
+ def _instant(text: str) -> datetime:
277
+ """The server's ISO text of a time, read as an aware UTC datetime:
278
+ `fromisoformat` takes a `Z` only from Python 3.11."""
279
+ return datetime.fromisoformat(text.replace("Z", "+00:00"))
280
+
281
+
282
+ def _named(row: Mapping[str, Any]) -> NamedStatement:
283
+ """A row of `statements` as a `NamedStatement`."""
284
+ last, held, after = row.get("last"), row.get("held"), row.get("after")
285
+ return NamedStatement(
286
+ name=row["name"],
287
+ version=tuple(row["version"]),
288
+ signature=row.get("signature"),
289
+ writes=row.get("writes"),
290
+ comment=row.get("comment"),
291
+ broken=row.get("broken"),
292
+ after=tuple(after) if after else None,
293
+ schedule=row.get("schedule"),
294
+ next=_instant(row["next"]) if row.get("next") else None,
295
+ last=(
296
+ LastRun(_instant(last["slot"]), last["outcome"], last.get("status"), last.get("reason"), last.get("skipped", 0))
297
+ if last
298
+ else None
299
+ ),
300
+ held=HeldSlot(_instant(held["slot"]), held["status"], _instant(held["since"])) if held else None,
301
+ hook=row.get("hook"),
302
+ armed=row.get("armed"),
303
+ )
304
+
305
+
306
+ class Retry(NamedTuple):
307
+ """How a client sends a request again: only where nothing ran, or where
308
+ running it again changes nothing.
309
+
310
+ Nothing ran: a 503, or a quota's 429, naming its wait -- sent after that
311
+ wait, never sooner, unless its body says statements of a batch were
312
+ made (`completed`), which the server never answers so; a connection
313
+ refused; a request not handed to the system whole. Running it again
314
+ changes nothing: a read the builder made, one sent near (`read=True` on
315
+ a client with `reads="nearest"`: `Fennec-Read: replica`, which every
316
+ server runs read-only), and a write under an `Idempotency-Key`, answered
317
+ from its key while the server keeps it (a day) -- these go again after a
318
+ reset or a dropped connection too. Nothing goes again after the client's
319
+ own `timeout`, past which the server may still run it, nor a refusal
320
+ naming no wait; `read=True` sent home is a claim nothing checks, and
321
+ goes once.
322
+
323
+ `attempts` tries at most, their waits -- the server's, or 0.25 s doubling,
324
+ 30% jitter on top -- within `max_wait` seconds; past either, the last
325
+ answer or error is the caller's. `Client(retry=False)` sends nothing
326
+ again."""
327
+
328
+ attempts: int = 3
329
+ max_wait: float = 10.0
330
+
331
+
332
+ def _retry(retry: Retry | bool | None) -> Retry | None:
333
+ if retry is True:
334
+ return Retry()
335
+ if not retry:
336
+ return None
337
+ if not isinstance(retry, Retry) or retry.attempts < 1 or retry.max_wait < 0:
338
+ raise ValueError(f"bad retry: {retry!r}")
339
+ return retry
340
+
341
+
342
+ def _wait_of(status: int, raw: bytes, header: str | None) -> float | None:
343
+ """The seconds a 503 or a 429 asks a client to wait, where it names any:
344
+ then nothing ran. To the millisecond where the body says it
345
+ (`retry_after_ms`), before `Retry-After` -- whole seconds -- and the
346
+ body's `retry_after`, which every 503 of the server's carries. None
347
+ where statements of a batch were made (`completed`): sent again whole,
348
+ they were made twice."""
349
+ if status not in (429, 503):
350
+ return None
351
+ try:
352
+ body = json.loads(raw)
353
+ except ValueError:
354
+ body = None
355
+ if not isinstance(body, dict):
356
+ body = {}
357
+ done = body.get("completed")
358
+ if isinstance(done, int) and done > 0:
359
+ return None
360
+ ms = body.get("retry_after_ms")
361
+ if isinstance(ms, (int, float)) and not isinstance(ms, bool) and ms > 0:
362
+ return ms / 1000
363
+ for said in (header, body.get("retry_after")):
364
+ try:
365
+ wait = float(said)
366
+ except (TypeError, ValueError):
367
+ continue
368
+ if 0 < wait < float("inf"):
369
+ return wait
370
+ return None
371
+
372
+
373
+ def _pause(retry: Retry | None, tries: int, waited: float, wait: float) -> float | None:
374
+ """How long to wait before try `tries + 1`, after `waited` seconds of
375
+ waits: the server's `wait`, or the backoff where longer, 30% jitter on
376
+ top, within what is left of `max_wait`; None where no try is left."""
377
+ base = max(wait, 0.25 * 2 ** (tries - 1))
378
+ if retry is None or tries >= retry.attempts or waited + base > retry.max_wait:
379
+ return None
380
+ return min(base * (1 + random.random() * 0.3), retry.max_wait - waited)
381
+
382
+
383
+ def _bound(sock: socket.socket, secs: float | None) -> None:
384
+ """Gives each of `sock`'s next waits `secs`, None for no bound, and
385
+ gives the request up where none is left. Within a millisecond either
386
+ way: a `settimeout` is a system call (0.3 us), and the one a quick
387
+ request left is as good as set."""
388
+ if secs is not None and secs <= 0:
389
+ raise TimeoutError("timed out")
390
+ was = sock.gettimeout()
391
+ if (was is None) != (secs is None) or (secs is not None and abs(was - secs) > 0.001):
392
+ sock.settimeout(secs)
393
+
394
+
395
+ # One refusal in an OSError's words, as asyncio joins every address's
396
+ # error when a host's addresses all fail ("Multiple exceptions: ...").
397
+ _REFUSAL = re.compile(rf"\[(?:Errno {errno.ECONNREFUSED}|WinError 1225)\] [^\[]*")
398
+
399
+
400
+ def _refused(e: BaseException) -> bool:
401
+ """Whether connecting was refused, at every address of the host's where
402
+ it has several: `asyncio.open_connection` tries each and, their words
403
+ differing by the address, raises one `OSError` joining them, the
404
+ refusals in it no `ConnectionRefusedError` any more. A request refused
405
+ so sent nothing, and any goes again."""
406
+ if isinstance(e, ConnectionRefusedError):
407
+ return True
408
+ said = str(e)
409
+ rest = said.removeprefix("Multiple exceptions: ")
410
+ if not isinstance(e, OSError) or rest == said:
411
+ return False
412
+ return all(_REFUSAL.fullmatch(part) for part in re.split(r", (?=\[(?:Errno|WinError) )", rest))
413
+
414
+
415
+ class _Shared:
416
+ """What a client and the copies `with_idempotency_key` makes of it
417
+ share, as Go's and .NET's copies share theirs: the change the last
418
+ answer from home left the database at, whether it was replayed, and
419
+ the furthest position an answer from home named, which a read near
420
+ waits for -- an answer that arrives late is not the last."""
421
+
422
+ __slots__ = ("seq", "replayed", "at", "_lock")
423
+
424
+ def __init__(self) -> None:
425
+ self.seq: int | None = None
426
+ self.replayed = False
427
+ self.at: tuple | None = None
428
+ # A client's threads share its copies' position.
429
+ self._lock = threading.Lock()
430
+
431
+ def answered(self, seq: int, history: str | None) -> None:
432
+ """An answer from home named `seq`, of `history`."""
433
+ p = _position(seq, history)
434
+ with self._lock:
435
+ self.seq = seq
436
+ if _later(p, self.at):
437
+ self.at = p
438
+
439
+ def told(self, raw: bytes, sent: tuple) -> None:
440
+ """A read near's 504, the read sent after `sent`: no copy held it,
441
+ home either -- a write lost in a failover. Home's position, lower or
442
+ not, is the client's from now on: kept, every read near would wait
443
+ it out for good. Taken only over `sent` -- a write answered
444
+ meanwhile is further, and set back the next read near would miss
445
+ it -- and never of an earlier term of the same lineage: a router
446
+ whose copy of the directory lags a failover sends the read to the
447
+ old primary, fenced, which names the branch it abandoned. Another
448
+ lineage's is a tenant made again under the name, which starts again
449
+ at term 1."""
450
+ try:
451
+ body = json.loads(raw)
452
+ except ValueError:
453
+ return
454
+ if isinstance(body, dict) and isinstance(body.get("seq"), int):
455
+ history = body.get("history")
456
+ p = _position(body["seq"], history if isinstance(history, str) else None)
457
+ with self._lock:
458
+ if self.at is sent and (p[0] >= sent[0] or _another(p, sent)):
459
+ self.at = p
460
+
461
+
462
+ def _position(seq: int, history: str | None) -> tuple:
463
+ """`(term, seq, history)`: a change, the history it is of
464
+ (`<term>-<id>`; none for a database that never had a replica) and that
465
+ history's term, one more at each promotion."""
466
+ term = 0
467
+ if history:
468
+ try:
469
+ term = int(history.split("-", 1)[0])
470
+ except ValueError:
471
+ term = 0
472
+ return (term, seq, history)
473
+
474
+
475
+ def _lineage(p: tuple) -> str | None:
476
+ """The lineage of position `p`: its history id's top half, which every
477
+ fork of the tenant keeps and a tenant made again under its name does
478
+ not."""
479
+ history = p[2]
480
+ if not history or "-" not in history:
481
+ return None
482
+ return history.split("-", 1)[1][:8] or None
483
+
484
+
485
+ def _another(p: tuple, q: tuple) -> bool:
486
+ """Whether `p` is of another lineage than `q`: a tenant made again."""
487
+ a, b = _lineage(p), _lineage(q)
488
+ return a is not None and b is not None and a != b
489
+
490
+
491
+ def _later(p: tuple, q: tuple | None) -> bool:
492
+ """Whether position `p` is past `q`: of another lineage -- home's word
493
+ on a tenant made again, whose terms start again -- or a later term of
494
+ the same, or further in one."""
495
+ return q is None or _another(p, q) or p[:2] > q[:2]
496
+
497
+
498
+ def _reads(reads: str) -> str:
499
+ if reads not in ("home", "nearest"):
500
+ raise ValueError(f"reads is 'home' or 'nearest' (got {reads!r})")
501
+ return reads
502
+
503
+
504
+ def _near(headers: dict, at: tuple | None, explicit: int | None) -> None:
505
+ """A read marked for a replica near the client, after `explicit` where
506
+ the caller named a change, and after its furthest position otherwise."""
507
+ headers["Fennec-Read"] = "replica"
508
+ if explicit is not None:
509
+ headers["Fennec-After"] = str(explicit)
510
+ elif at is not None:
511
+ headers["Fennec-After"] = str(at[1])
512
+ if at[2]:
513
+ headers["Fennec-History"] = at[2]
514
+
515
+
516
+ class Client:
517
+ """fennec-server's HTTP endpoint, one statement a request.
518
+
519
+ A statement goes over a connection kept alive, one a thread: through
520
+ `urlopen`, which opens one a request, a read by id took 152 us against
521
+ 57 over the one connection, the server starting a thread for each.
522
+
523
+ `timeout` bounds the seconds to an answer's head -- a `wait` holds it
524
+ that much longer -- and those between the pieces of its body, which is
525
+ read for as long as it keeps coming; None, never. A statement that runs
526
+ long before it answers -- a `compact`, a `create index` -- is given a
527
+ client with a longer `timeout`: past it, it may still run, and is never
528
+ sent again. `retry` sends a request again where nothing ran or running
529
+ it twice is harmless (`Retry`); `False`, never.
530
+ """
531
+
532
+ def __init__(
533
+ self,
534
+ url: str = "http://127.0.0.1:8080",
535
+ token: str | None = None,
536
+ timeout: float | None = 30.0,
537
+ reads: str = "home",
538
+ retry: Retry | bool = True,
539
+ ):
540
+ parts = urllib.parse.urlsplit(url)
541
+ if parts.scheme not in ("http", "https"):
542
+ raise ValueError(f"not an http(s) URL: {url!r}")
543
+ self.url = url.rstrip("/")
544
+ self.token = token
545
+ self.timeout = timeout
546
+ self.retry = _retry(retry)
547
+ # "nearest": reads marked for a read replica near the client.
548
+ self.reads = _reads(reads)
549
+ self.idempotency_key: str | None = None
550
+ self._shared = _Shared()
551
+ self._host = parts.hostname or "127.0.0.1"
552
+ self._port = parts.port or (443 if parts.scheme == "https" else 80)
553
+ self._tls = parts.scheme == "https"
554
+ self._base = parts.path.rstrip("/")
555
+ self._local = threading.local()
556
+
557
+ @property
558
+ def seq(self) -> int | None:
559
+ """The change the last write through this client, or a copy of it,
560
+ left the database at (`Fennec-Seq`): a read on a replica passed it as
561
+ `after` waits for that write."""
562
+ return self._shared.seq
563
+
564
+ @seq.setter
565
+ def seq(self, value: int | None) -> None:
566
+ self._shared.seq = value
567
+
568
+ @property
569
+ def replayed(self) -> bool:
570
+ """Whether the last write's answer was the one kept for its
571
+ `Idempotency-Key`: the write had run before, and did not again."""
572
+ return self._shared.replayed
573
+
574
+ def with_idempotency_key(self, key: str) -> "Client":
575
+ """A copy of the client whose writes -- `query`, `batch` and the
576
+ builder's -- carry `key` as their `Idempotency-Key`: sent again
577
+ after a timeout, a write runs once. It shares this client's
578
+ connections and `seq`. One key per write; the same key with
579
+ another request is refused (422)."""
580
+ if not isinstance(key, str) or not key:
581
+ raise ValueError("an idempotency key is a text, not empty")
582
+ copy = object.__new__(Client)
583
+ copy.__dict__.update(self.__dict__)
584
+ copy.idempotency_key = key
585
+ return copy
586
+
587
+ def collection(self, name: str) -> Query:
588
+ """The query builder over collection `name`: chain `select`, `where`,
589
+ `near`, `order`, `limit` ... and end with `rows()`, `first()`,
590
+ `count()`, or a write -- `insert`, `update`, `delete`."""
591
+ return Query(name, self)
592
+
593
+ def call(
594
+ self,
595
+ name: str,
596
+ args: Mapping[str, Any] | None = None,
597
+ *,
598
+ claims: Mapping[str, Any] | None = None,
599
+ idempotency_key: str | None = None,
600
+ wait: float | None = None,
601
+ read: bool = False,
602
+ ) -> Any:
603
+ """Calls the named statement `name` with `args`, a mapping of values
604
+ by name: `call <name> $1` over `/query`, sent and answered as
605
+ `query` does it -- the last statement of its body, rows or
606
+ `{"affected": n}`. A refusal is a `FennecError` as any statement's:
607
+ `status` 404 for no such statement or none granted, 403 for a claim
608
+ the token lacks, 400 for an argument missing, extra or of another
609
+ type, 412 for a `require` not met (its `reason`), 409 for a unique
610
+ value taken or a broken statement. `claims` is `as {...}`, the
611
+ claims the call is made with: the server's token alone may say them,
612
+ a scoped token's are its own. `idempotency_key` and `wait` are
613
+ `query`'s. `read=True` says the statement only reads, and on a
614
+ client with `reads="nearest"` it goes near, after this client's
615
+ writes; one that writes is refused there (403), never landed. Not
616
+ said, a call goes home, as every text not known to read does."""
617
+ text, params = Call(name, args, claims=claims).to_call()
618
+ return self.query(text, params, idempotency_key=idempotency_key, wait=wait, read=read)
619
+
620
+ def statements(self) -> list[NamedStatement]:
621
+ """Each named statement the database holds, as the server lists them
622
+ to its own token (`statements`) -- a scoped one is refused (403):
623
+ its signature, whether it writes, its comment, why the schema broke
624
+ it, what a binding is bound to, a scheduled one's clock, and a
625
+ webhook consumer's hook and whether the server holds its secret.
626
+ `call` sends a webhook consumer a test event, the event its
627
+ argument, run as a delivery is and keeping no key."""
628
+ return [_named(r) for r in self.query("statements")]
629
+
630
+ def query(
631
+ self,
632
+ fennecql: str,
633
+ params: Sequence[Any] | None = None,
634
+ *,
635
+ after: int | None = None,
636
+ idempotency_key: str | None = None,
637
+ wait: float | None = None,
638
+ read: bool = False,
639
+ ) -> Any:
640
+ """Runs one FennecQL statement. Values go in as `$1`, `$2`, ... and
641
+ never into the text. With `after` -- a primary's `seq` -- a replica
642
+ answers once it holds that write. With `idempotency_key` a write
643
+ runs once however often it is sent (`replayed`). With `wait` a
644
+ `set` or `del` that writes nothing is held at the server until it
645
+ can, or that many seconds -- a claim waiting for a job. `read=True`
646
+ says the statement only reads: on a client with `reads="nearest"`,
647
+ a router near the tenant's read replica answers it there, after
648
+ this client's writes. A read answers its rows, a list; one with a
649
+ `facet` clause a `Rows`, the counts as its `facets`."""
650
+ body = {"query": fennecql, "params": list(params or [])}
651
+ return _faceted(
652
+ self._post(
653
+ "/query",
654
+ json.dumps(body).encode(),
655
+ "application/json",
656
+ after,
657
+ key=idempotency_key,
658
+ wait=wait,
659
+ read=read,
660
+ )
661
+ )
662
+
663
+ def batch(
664
+ self,
665
+ statements: Iterable[tuple[str, Sequence[Any]]],
666
+ *,
667
+ idempotency_key: str | None = None,
668
+ wait: float | None = None,
669
+ read: bool = False,
670
+ ) -> Any:
671
+ """Runs statements in order under one write lock, as one block:
672
+ their writes -- a create, a drop or a `create index` among them --
673
+ all land or, at the first error, none of them do: a `FennecError`
674
+ whose `at` is the statement that stopped it. A batch holding a
675
+ `compact` runs each statement on its own, and what ran before an
676
+ error stays (`completed`). Answers `{"ok": n, "results": [...]}`,
677
+ a statement's `{"affected": n}` or `{"rows": [...]}` each, and
678
+ leaves the change it wrote at in `seq`. With `idempotency_key` it
679
+ lands once however often it is sent (`replayed`); a batch of reads
680
+ alone takes no key. With `wait` one whose `set`s and `del`s write
681
+ nothing is held until they can, as `query`'s; `read=True`, a batch
682
+ of reads alone, goes near as `query`'s does."""
683
+ lines = [json.dumps({"query": q, "params": list(p)}) for q, p in statements]
684
+ return self._post(
685
+ "/batch",
686
+ "\n".join(lines).encode(),
687
+ "application/x-ndjson",
688
+ key=idempotency_key,
689
+ wait=wait,
690
+ read=read,
691
+ )
692
+
693
+ def schema(
694
+ self, fennecql: str, migrations: Sequence[Any] | None = None, *, migrate: bool = False
695
+ ) -> dict:
696
+ """Checks the server's schema against `fennecql` -- `create
697
+ collection` and `create index` statements, a `schema.fennecql` file
698
+ -- and returns the engine's plan. The server owns its schema: it is
699
+ compared, and nothing applied, unless `migrate` -- with the server's
700
+ token -- runs the migrations it has not recorded, in order, and adds
701
+ what only adds, all one block. Raises `SchemaError` naming every
702
+ difference that would lose data or could mean two things."""
703
+ body = {"format": 1, "fennecql": fennecql, "migrations": list(migrations or [])}
704
+ path = "/_schema/apply" if migrate else "/_schema/plan?mode=follow"
705
+ out = self._post(path, json.dumps(body).encode(), "application/json", accept=409)
706
+ if out["refusals"]:
707
+ raise SchemaError(out["refusals"], 409)
708
+ return out
709
+
710
+ def changes(
711
+ self,
712
+ since: int | None = None,
713
+ *,
714
+ consumer: str | None = None,
715
+ limit: int | None = None,
716
+ wait: float | None = None,
717
+ ) -> Changes:
718
+ """The writes on the server's disk after `since` -- or after where
719
+ `consumer` stands, or from now with neither -- at most `limit`, each
720
+ `{"seq", "at", "collection", "op", "id", "doc"}`. With `wait`, an
721
+ answer with nothing in it waits that many seconds for a write."""
722
+ query = {"since": since, "consumer": consumer, "limit": limit}
723
+ if wait is not None:
724
+ query["wait"] = int(wait * 1000)
725
+ q = urllib.parse.urlencode({k: v for k, v in query.items() if v is not None})
726
+ status, got, raw = self._send(
727
+ "GET", f"{self._base}/_changes?{q}", None, self._auth(), _more(self.timeout, wait), True
728
+ )
729
+ if not 200 <= status < 300:
730
+ _answer(status, raw)
731
+ nxt = int(got.get("Fennec-Next", since or 0))
732
+ seq = max(int(got.get("Fennec-Seq", 0)), nxt)
733
+ return Changes([json.loads(line) for line in raw.splitlines() if line], nxt, seq)
734
+
735
+ def consumer(self, name: str, since: int | None = None) -> dict:
736
+ """Makes `name` a consumer at the last write on disk, or at `since`,
737
+ and returns where it stands, `{"name", "since"}`. One that is there
738
+ already stays where it stands -- `since` is a new consumer's start
739
+ -- so a program can call this every time it starts and go on from
740
+ where it stopped; a `since` ahead of where it stands is a
741
+ `FennecError` (409), a commit sent the old way. `move_consumer`
742
+ moves one."""
743
+ body = json.dumps({} if since is None else {"since": since}).encode()
744
+ return self._post(f"/_changes/consumers/{_seg(name)}", body, "application/json")
745
+
746
+ def move_consumer(self, name: str, since: int) -> dict:
747
+ """Moves `name` to `since`: on, once it has done with every write up
748
+ to it (a commit), or back, to read from there again, and returns
749
+ where it stands after -- where it was, for a commit past nothing but
750
+ consumers' own commits, which reads the same writes. A consumer that
751
+ is not there is a `FennecError` (404): `consumer` makes one."""
752
+ body = json.dumps({"since": since}).encode()
753
+ headers = {**self._auth(), "Content-Type": "application/json"}
754
+ status, _, raw = self._send("PATCH", f"{self._base}/_changes/consumers/{_seg(name)}", body, headers)
755
+ return _answer(status, raw)
756
+
757
+ def consumers(self) -> list:
758
+ """Each consumer: its `name`, `since`, and how many writes `behind`."""
759
+ status, _, raw = self._send("GET", f"{self._base}/_changes/consumers", None, self._auth(), twice=True)
760
+ return _answer(status, raw)
761
+
762
+ def forget_consumer(self, name: str) -> None:
763
+ status, _, raw = self._send("DELETE", f"{self._base}/_changes/consumers/{_seg(name)}", None, self._auth())
764
+ _answer(status, raw)
765
+
766
+ def follow(self, consumer: str, *, limit: int = 1000, wait: float = 10.0) -> Iterator[list]:
767
+ """Each batch of writes for `consumer`, made if it is new, committed
768
+ once the loop asks for the next: a batch the loop did not finish is
769
+ read again, so each write comes at least once. Started again, it
770
+ goes on from the last batch committed."""
771
+ self.consumer(consumer)
772
+ while True:
773
+ got = self.changes(consumer=consumer, limit=limit, wait=wait)
774
+ # Nothing read, nothing committed: a commit is a write, which
775
+ # the stream passes over, and committing past it every time
776
+ # would write once a wait.
777
+ if got.writes:
778
+ yield got.writes
779
+ self.move_consumer(consumer, got.next)
780
+
781
+ def _auth(self) -> dict:
782
+ return {"Authorization": f"Bearer {self.token}"} if self.token else {}
783
+
784
+ def _post(
785
+ self,
786
+ path: str,
787
+ body: bytes,
788
+ content_type: str,
789
+ after: int | None = None,
790
+ accept: int = 0,
791
+ key: str | None = None,
792
+ wait: float | None = None,
793
+ read: bool = False,
794
+ ) -> Any:
795
+ headers = {"Content-Type": content_type}
796
+ if self.token:
797
+ headers["Authorization"] = f"Bearer {self.token}"
798
+ near = read and self.reads == "nearest"
799
+ if near:
800
+ sent = self._shared.at
801
+ _near(headers, sent, after)
802
+ elif after is not None:
803
+ headers["Fennec-After"] = str(after)
804
+ key = key or self.idempotency_key
805
+ if key:
806
+ headers["Idempotency-Key"] = key
807
+ if wait:
808
+ headers["Fennec-Wait"] = _wait_ms(wait)
809
+ timeout = _more(self.timeout, wait)
810
+ # What may run twice, sent again after a reset too: a read the
811
+ # builder made (`_BUILT`), one sent near, which every server runs
812
+ # read-only (`Fennec-Read`), and a write under a key. The caller's
813
+ # `read=True` is a claim nothing checks at home: a call it names
814
+ # may have been made to write since.
815
+ twice = read is _BUILT or bool(key)
816
+ status, got, raw = self._send("POST", self._base + path, body, headers, timeout, twice or near)
817
+ # 504: no copy the router reached holds that write in the wait --
818
+ # the write lost in a failover of home, which says where it stands
819
+ # now: the client's position from here on. Asked home with no mark,
820
+ # the read is answered with what home holds.
821
+ if near and status == 504 and "Fennec-After" in headers:
822
+ # A change the caller named is not the client's position.
823
+ if after is None and sent is not None:
824
+ self._shared.told(raw, sent)
825
+ for h in ("Fennec-Read", "Fennec-After", "Fennec-History"):
826
+ headers.pop(h, None)
827
+ near = False
828
+ status, got, raw = self._send("POST", self._base + path, body, headers, timeout, twice)
829
+ if 200 <= status < 300:
830
+ # A replayed batch's answer carries no `Fennec-Seq`: `seq` stays
831
+ # where the last write that sent one left it. A read near's
832
+ # answer may be a replica's: it moves nothing.
833
+ seq = got.get("Fennec-Seq")
834
+ if seq is not None and not near:
835
+ self._shared.answered(int(seq), got.get("Fennec-History"))
836
+ self._shared.replayed = got.get("Idempotent-Replayed") == "true"
837
+ # An answer that is no error: a schema's refusals (409).
838
+ if status == accept:
839
+ return json.loads(raw)
840
+ return _answer(status, raw)
841
+
842
+ def _send(
843
+ self,
844
+ method: str,
845
+ path: str,
846
+ body: bytes | None,
847
+ headers: dict,
848
+ timeout: float | None = None,
849
+ twice: bool = False,
850
+ ) -> tuple:
851
+ """A request, its status, headers and body, sent again by the rule
852
+ the router keeps: only what cannot have run, or may run twice. An
853
+ answer naming its wait -- a 503, a quota's 429 -- ran nothing, unless
854
+ it says statements of a batch were made (`_wait_of`), nor did a
855
+ request whose connection was refused or that was not handed to the
856
+ system whole: any request goes again, after that wait and never
857
+ sooner. `twice` -- a read the builder made or sent near, a write
858
+ under a key -- goes again after a reset or a dropped connection too,
859
+ which a write may have landed before. Nothing goes again after the
860
+ client's own timeout, past which the server may still run it, nor a
861
+ refusal naming no wait; past the `Retry`'s tries or its waits, the
862
+ last answer or error is the caller's."""
863
+ tries, waited = 0, 0.0
864
+ while True:
865
+ tries += 1
866
+ error = None
867
+ try:
868
+ status, got, raw = self._once(method, path, body, headers, timeout)
869
+ # Headers asked of a 429 or a 503 alone: an `HTTPMessage`'s
870
+ # `get` walks every header.
871
+ wait = _wait_of(status, raw, got.get("Retry-After")) if status in (429, 503) else None
872
+ if wait is None:
873
+ return status, got, raw
874
+ except TimeoutError:
875
+ # The client's own: a `compact`, a long read may still be
876
+ # running, and three tries ran a read three times.
877
+ raise
878
+ except _Unsent as e:
879
+ error, wait = e.__cause__, 0.0
880
+ except (OSError, http.client.HTTPException) as e:
881
+ if not (twice or isinstance(e, ConnectionRefusedError)):
882
+ raise
883
+ error, wait = e, 0.0
884
+ pause = _pause(self.retry, tries, waited, wait)
885
+ if pause is None:
886
+ if error is not None:
887
+ raise error
888
+ return status, got, raw
889
+ waited += pause
890
+ time.sleep(pause)
891
+
892
+ def _once(
893
+ self, method: str, path: str, body: bytes | None, headers: dict, timeout: float | None
894
+ ) -> tuple:
895
+ """One try over this thread's connection: the connect and the
896
+ answer's head within `timeout` -- each wait given what is left of it
897
+ -- and each piece of its body within `self.timeout` of the last, so
898
+ a long answer is read for as long as it comes and one that stops is
899
+ given up. Bounded whole, a long read failed however steadily it
900
+ came. A kept connection the server closed while it was idle is found
901
+ so before the request goes -- its socket reads as ready, the end of
902
+ the stream -- and a new one is opened in its place: a try on it
903
+ would fail as though the request may have run."""
904
+ limit = self.timeout if timeout is None else timeout
905
+ end = None if limit is None else time.monotonic() + limit
906
+
907
+ def left() -> float | None:
908
+ return None if end is None else end - time.monotonic()
909
+
910
+ conn = getattr(self._local, "conn", None)
911
+ if conn is not None and _closed(conn):
912
+ conn.close()
913
+ conn = None
914
+ if conn is None:
915
+ conn = self._connect()
916
+ try:
917
+ if conn.sock is None:
918
+ conn.timeout = limit
919
+ conn.connect()
920
+ sock = conn.sock
921
+ _bound(sock, left())
922
+ try:
923
+ conn.request(method, path, body, headers)
924
+ except TimeoutError:
925
+ # A server that stopped reading: sent again, it would wait as
926
+ # long ([`TimeoutError`] in `_send`).
927
+ raise
928
+ except OSError as e:
929
+ # Not handed to the system whole, as the router judges it:
930
+ # the server read no request to run.
931
+ raise _Unsent(e) from e
932
+ # A small request goes at once, the bound before it as good for
933
+ # the head; a large one may take its time going.
934
+ if body is not None and len(body) > 65536:
935
+ _bound(sock, left())
936
+ resp = conn.getresponse()
937
+ # `read` gives each wait for a piece of the body the socket's
938
+ # timeout: the timeout between pieces.
939
+ _bound(sock, self.timeout)
940
+ raw = resp.read()
941
+ except BaseException:
942
+ conn.close()
943
+ self._local.conn = None
944
+ raise
945
+ if resp.will_close:
946
+ conn.close()
947
+ self._local.conn = None
948
+ else:
949
+ self._local.conn = conn
950
+ return resp.status, resp.msg, raw
951
+
952
+ def _connect(self) -> http.client.HTTPConnection:
953
+ if self._tls:
954
+ return http.client.HTTPSConnection(
955
+ self._host, self._port, timeout=self.timeout, context=ssl.create_default_context()
956
+ )
957
+ return http.client.HTTPConnection(self._host, self._port, timeout=self.timeout)
958
+
959
+ def close(self) -> None:
960
+ """Closes this thread's connection; the next request opens another."""
961
+ conn = getattr(self._local, "conn", None)
962
+ if conn is not None:
963
+ conn.close()
964
+ self._local.conn = None
965
+
966
+
967
+ class _Unsent(Exception):
968
+ """A request not handed to the system whole: its cause, the error that
969
+ stopped it, is what the caller meets once no try is left."""
970
+
971
+
972
+ def _more(timeout: float | None, wait: float | None) -> float | None:
973
+ """A request's timeout, and the `wait` the server may hold it on top."""
974
+ return None if timeout is None else timeout + (wait or 0)
975
+
976
+
977
+ def _wait_ms(wait: float) -> str:
978
+ """`Fennec-Wait` for `wait` seconds: milliseconds, as the header is."""
979
+ if isinstance(wait, bool) or not isinstance(wait, (int, float)) or wait < 0:
980
+ raise ValueError(f"wait is seconds, a number from 0 (got {wait!r})")
981
+ return str(int(wait * 1000))
982
+
983
+
984
+ def _seg(name: str) -> str:
985
+ return urllib.parse.quote(name, safe="")
986
+
987
+
988
+ def _faceted(answer: Any) -> Any:
989
+ """A `/query` answer as it came, but for a query that asked facets:
990
+ the server sends `{"rows": [...], "facets": {...}}` for it rather than
991
+ the bare array, and that becomes `Rows` -- a list of the rows, as every
992
+ other read answers, with the counts as its `facets`. A `/batch` answer
993
+ is left as it came, each read's result `{"rows": ..., "facets": ...}`."""
994
+ if isinstance(answer, dict) and "facets" in answer and isinstance(answer.get("rows"), list):
995
+ return _rows(answer)
996
+ return answer
997
+
998
+
999
+ def _closed(conn: http.client.HTTPConnection) -> bool:
1000
+ """Whether a kept connection can no longer take a request: the server
1001
+ closed it (its socket reads as ready, at the end of the stream) or sent
1002
+ what no request asked for. Between requests nothing is due on it."""
1003
+ sock = conn.sock
1004
+ if sock is None:
1005
+ return True
1006
+ try:
1007
+ ready, _, _ = select.select([sock], [], [], 0)
1008
+ except (OSError, ValueError):
1009
+ return True
1010
+ return bool(ready)
1011
+
1012
+
1013
+ def _answer(status: int, raw: bytes) -> Any:
1014
+ """A response's JSON, or the error it says, as `Client` raises it."""
1015
+ if 200 <= status < 300:
1016
+ return json.loads(raw) if raw else None
1017
+ at = completed = quota = retry_after = outcome = seq = history = reason = binding = None
1018
+ try:
1019
+ body = json.loads(raw)
1020
+ message = body.get("error", raw.decode(errors="replace"))
1021
+ at, completed = body.get("at"), body.get("completed")
1022
+ # What a `require n else '<reason>'` that refused said, a batch's
1023
+ # stop's beside its `at`.
1024
+ reason = body.get("reason")
1025
+ # The statement bound to the write that refused it, by name.
1026
+ binding = body.get("binding")
1027
+ # A quota's refusal says which, and the wait its `Retry-After`
1028
+ # says -- a quota's 429, a move's 503 -- in the body too: what is
1029
+ # read here has no headers. To the millisecond where it says it: a
1030
+ # bucket of 600 rows a second refills a row in 1.7 ms.
1031
+ quota, retry_after = body.get("quota"), body.get("retry_after")
1032
+ ms = body.get("retry_after_ms")
1033
+ if isinstance(ms, (int, float)) and ms > 0:
1034
+ retry_after = ms / 1000
1035
+ # A write a tenant's node answered after its lease lapsed: neither
1036
+ # made nor refused, its position for a read once the tenant settles.
1037
+ if body.get("outcome") == "unknown":
1038
+ outcome, seq, history = "unknown", body.get("seq"), body.get("history")
1039
+ except (ValueError, AttributeError):
1040
+ message = raw.decode(errors="replace")
1041
+ raise FennecError(
1042
+ message,
1043
+ status,
1044
+ at=at,
1045
+ completed=completed,
1046
+ quota=quota,
1047
+ retry_after=retry_after,
1048
+ outcome=outcome,
1049
+ seq=seq,
1050
+ history=history,
1051
+ reason=reason,
1052
+ binding=binding,
1053
+ )
1054
+
1055
+
1056
+ class AsyncClient:
1057
+ """fennec-server's HTTP endpoint over asyncio: `Client`'s calls, awaited.
1058
+
1059
+ One connection, kept alive and opened again when the server closed it;
1060
+ calls made at once on one client go one after another over it, as they
1061
+ would over one `Client` -- a client each for requests side by side.
1062
+ `https://` for a server behind a TLS terminator. The standard library
1063
+ alone: asyncio's streams and a request written out by hand, the
1064
+ endpoint's answers being a status, a length and JSON. `timeout` and
1065
+ `retry` are `Client`'s.
1066
+ """
1067
+
1068
+ def __init__(
1069
+ self,
1070
+ url: str = "http://127.0.0.1:8080",
1071
+ token: str | None = None,
1072
+ timeout: float | None = 30.0,
1073
+ reads: str = "home",
1074
+ retry: Retry | bool = True,
1075
+ ):
1076
+ parts = urllib.parse.urlsplit(url)
1077
+ if parts.scheme not in ("http", "https"):
1078
+ raise ValueError(f"not an http(s) URL: {url!r}")
1079
+ self.url = url.rstrip("/")
1080
+ self.token = token
1081
+ self.timeout = timeout
1082
+ self.retry = _retry(retry)
1083
+ self.reads = _reads(reads)
1084
+ self._host = parts.hostname or "127.0.0.1"
1085
+ self._port = parts.port or (443 if parts.scheme == "https" else 80)
1086
+ self._tls = ssl.create_default_context() if parts.scheme == "https" else None
1087
+ self._base = parts.path.rstrip("/")
1088
+ self.idempotency_key: str | None = None
1089
+ # The connection, its lock, `seq` and `replayed`, shared with the
1090
+ # copies `with_idempotency_key` makes: one connection still.
1091
+ self._s = _AsyncShared()
1092
+
1093
+ @property
1094
+ def seq(self) -> int | None:
1095
+ """The change the last write left the database at, as `Client.seq`."""
1096
+ return self._s.seq
1097
+
1098
+ @property
1099
+ def replayed(self) -> bool:
1100
+ """Whether the last write's answer was replayed, as `Client.replayed`."""
1101
+ return self._s.replayed
1102
+
1103
+ def with_idempotency_key(self, key: str) -> "AsyncClient":
1104
+ """A copy whose writes carry `key`, as `Client.with_idempotency_key`,
1105
+ over this client's connection."""
1106
+ if not isinstance(key, str) or not key:
1107
+ raise ValueError("an idempotency key is a text, not empty")
1108
+ copy = object.__new__(AsyncClient)
1109
+ copy.__dict__.update(self.__dict__)
1110
+ copy.idempotency_key = key
1111
+ return copy
1112
+
1113
+ def collection(self, name: str) -> AsyncQuery:
1114
+ """The query builder, as `Client.collection`, its endpoints awaited."""
1115
+ return AsyncQuery(name, self)
1116
+
1117
+ async def call(
1118
+ self,
1119
+ name: str,
1120
+ args: Mapping[str, Any] | None = None,
1121
+ *,
1122
+ claims: Mapping[str, Any] | None = None,
1123
+ idempotency_key: str | None = None,
1124
+ wait: float | None = None,
1125
+ read: bool = False,
1126
+ ) -> Any:
1127
+ """A named statement's call, as `Client.call`, awaited; a held one is
1128
+ cancelled as any task is, its connection closed with it."""
1129
+ text, params = Call(name, args, claims=claims).to_call()
1130
+ return await self.query(text, params, idempotency_key=idempotency_key, wait=wait, read=read)
1131
+
1132
+ async def statements(self) -> list[NamedStatement]:
1133
+ """The named statements the database holds, as `Client.statements`."""
1134
+ return [_named(r) for r in await self.query("statements")]
1135
+
1136
+ async def query(
1137
+ self,
1138
+ fennecql: str,
1139
+ params: Sequence[Any] | None = None,
1140
+ *,
1141
+ idempotency_key: str | None = None,
1142
+ wait: float | None = None,
1143
+ read: bool = False,
1144
+ ) -> Any:
1145
+ """One FennecQL statement, as `Client.query`; a held one is
1146
+ cancelled as any task is, its connection closed with it."""
1147
+ body = {"query": fennecql, "params": list(params or [])}
1148
+ return _faceted(
1149
+ await self._post(
1150
+ "/query", json.dumps(body).encode(), "application/json", idempotency_key, wait, read
1151
+ )
1152
+ )
1153
+
1154
+ async def batch(
1155
+ self,
1156
+ statements: Iterable[tuple[str, Sequence[Any]]],
1157
+ *,
1158
+ idempotency_key: str | None = None,
1159
+ wait: float | None = None,
1160
+ read: bool = False,
1161
+ ) -> Any:
1162
+ """Statements under one write lock, as one block, as `Client.batch`."""
1163
+ lines = [json.dumps({"query": q, "params": list(p)}) for q, p in statements]
1164
+ return await self._post(
1165
+ "/batch", "\n".join(lines).encode(), "application/x-ndjson", idempotency_key, wait, read
1166
+ )
1167
+
1168
+ async def close(self) -> None:
1169
+ if self._s.conn:
1170
+ _, writer = self._s.conn
1171
+ self._s.conn = None
1172
+ writer.close()
1173
+ try:
1174
+ await writer.wait_closed()
1175
+ except OSError:
1176
+ pass
1177
+
1178
+ async def __aenter__(self) -> "AsyncClient":
1179
+ return self
1180
+
1181
+ async def __aexit__(self, *exc: Any) -> None:
1182
+ await self.close()
1183
+
1184
+ async def _post(
1185
+ self,
1186
+ path: str,
1187
+ body: bytes,
1188
+ content_type: str,
1189
+ key: str | None = None,
1190
+ wait: float | None = None,
1191
+ read: bool = False,
1192
+ ) -> Any:
1193
+ key = key or self.idempotency_key
1194
+ held = _wait_ms(wait) if wait else None
1195
+ # As `Client._post`'s: what may run twice -- a read the builder made,
1196
+ # one sent near, a write under a key -- goes again after a reset too.
1197
+ twice = read is _BUILT or bool(key)
1198
+ limit = _more(self.timeout, wait)
1199
+ async with self._s.lock:
1200
+ # A read near, as `Client`'s: marked, after this client's
1201
+ # furthest position, and asked home on a 504, taking home's.
1202
+ near: dict = {}
1203
+ if read and self.reads == "nearest":
1204
+ sent = self._s.at
1205
+ _near(near, sent, None)
1206
+ tries, waited = 0, 0.0
1207
+ while True:
1208
+ tries += 1
1209
+ error = None
1210
+ try:
1211
+ status, headers, raw = await self._try(path, body, content_type, key, held, near, limit)
1212
+ if raw is None:
1213
+ raw = await self._body(headers)
1214
+ if status == 504 and "Fennec-After" in near:
1215
+ self._s.told(raw, sent)
1216
+ near = {}
1217
+ status, headers, raw = await self._try(path, body, content_type, key, held, near, limit)
1218
+ if raw is None:
1219
+ raw = await self._body(headers)
1220
+ wait_s = _wait_of(status, raw, headers.get("retry-after"))
1221
+ if wait_s is None:
1222
+ if 200 <= status < 300:
1223
+ if "fennec-seq" in headers and not near:
1224
+ self._s.answered(int(headers["fennec-seq"]), headers.get("fennec-history"))
1225
+ self._s.replayed = headers.get("idempotent-replayed") == "true"
1226
+ return _answer(status, raw)
1227
+ except (TimeoutError, asyncio.TimeoutError):
1228
+ # The client's own, as `Client._send`'s: never again.
1229
+ await self.close()
1230
+ raise
1231
+ except (OSError, asyncio.IncompleteReadError) as e:
1232
+ await self.close()
1233
+ # Refused, at every address of the host's: nothing went,
1234
+ # and any request goes again.
1235
+ if not (twice or near or _refused(e)):
1236
+ raise
1237
+ error, wait_s = e, 0.0
1238
+ except asyncio.CancelledError:
1239
+ # Cancelled under way -- a held claim given up: its
1240
+ # answer, still to come, must not be the next request's.
1241
+ self._drop()
1242
+ raise
1243
+ pause = _pause(self.retry, tries, waited, wait_s)
1244
+ if pause is None:
1245
+ if error is not None:
1246
+ raise error
1247
+ return _answer(status, raw)
1248
+ waited += pause
1249
+ await asyncio.sleep(pause)
1250
+
1251
+ def _try(
1252
+ self,
1253
+ path: str,
1254
+ body: bytes,
1255
+ content_type: str,
1256
+ key: str | None,
1257
+ held: str | None,
1258
+ extra: dict,
1259
+ limit: float | None,
1260
+ ) -> Any:
1261
+ """One try, to be awaited: the connect, the answer's head and a body
1262
+ in hand with it within `limit` -- a body still to come is left to
1263
+ `_body`, None in its place. A kept connection the server closed
1264
+ while it was idle is let go of before the request goes, as `Client`
1265
+ finds one: a try on it would fail as one that may have run. Not a
1266
+ coroutine of its own: one more a request cost 0.4 us."""
1267
+ if self._s.conn is not None and _gone(*self._s.conn):
1268
+ self._drop()
1269
+ return asyncio.wait_for(self._exchange(path, body, content_type, key, held, extra), limit)
1270
+
1271
+ def _drop(self) -> None:
1272
+ if self._s.conn:
1273
+ _, writer = self._s.conn
1274
+ self._s.conn = None
1275
+ writer.close()
1276
+
1277
+ async def _exchange(
1278
+ self,
1279
+ path: str,
1280
+ body: bytes,
1281
+ content_type: str,
1282
+ key: str | None,
1283
+ held: str | None,
1284
+ extra: dict,
1285
+ ) -> tuple[int, dict, bytes | None]:
1286
+ """The request sent and the answer's head read (`_head`), and its
1287
+ body where the loop holds it whole already -- a head and a body the
1288
+ server wrote at once, as it writes every answer: nothing to wait
1289
+ for. Any other body is `_body`'s, read past the head's bound a piece
1290
+ at a time: one more coroutine and a timer the loop holds cost every
1291
+ request 3.5 us, 6%."""
1292
+ status, headers = await self._head(path, body, content_type, key, held, extra)
1293
+ n = headers.get("content-length")
1294
+ reader = self._s.conn[0]
1295
+ # CPython's buffer, and uvloop's, which runs asyncio's streams: a
1296
+ # reader without one has its body read by `_body`.
1297
+ if n is not None and len(getattr(reader, "_buffer", b"")) >= int(n):
1298
+ raw = await reader.readexactly(int(n))
1299
+ if headers.get("connection", "").lower() == "close":
1300
+ await self.close()
1301
+ return status, headers, raw
1302
+ return status, headers, None
1303
+
1304
+ async def _body(self, headers: dict) -> bytes:
1305
+ """The body of the answer whose `headers` were read, each piece
1306
+ within the timeout of the last (`_Idle`): a long answer is read for
1307
+ as long as it comes, one that stops is given up, its connection
1308
+ with it."""
1309
+ reader, writer = self._s.conn
1310
+ idle = _Idle(writer, self.timeout)
1311
+ try:
1312
+ if "content-length" in headers:
1313
+ raw = await _exactly(reader, int(headers["content-length"]), idle)
1314
+ elif headers.get("transfer-encoding", "").lower() == "chunked":
1315
+ parts = []
1316
+ while True:
1317
+ idle.tick()
1318
+ size = int((await reader.readuntil(b"\r\n")).split(b";")[0], 16)
1319
+ if not size:
1320
+ break
1321
+ parts.append(await _exactly(reader, size + 2, idle))
1322
+ idle.tick()
1323
+ await reader.readuntil(b"\r\n")
1324
+ raw = b"".join(p[:-2] for p in parts)
1325
+ else:
1326
+ parts = []
1327
+ while True:
1328
+ idle.tick()
1329
+ part = await reader.read(1 << 16)
1330
+ if not part:
1331
+ break
1332
+ parts.append(part)
1333
+ raw = b"".join(parts)
1334
+ self._s.conn = None
1335
+ except (OSError, asyncio.IncompleteReadError):
1336
+ if idle.late:
1337
+ raise asyncio.TimeoutError(f"no piece of the answer within {self.timeout} s") from None
1338
+ raise
1339
+ finally:
1340
+ idle.stop()
1341
+ if headers.get("connection", "").lower() == "close" and self._s.conn:
1342
+ await self.close()
1343
+ return raw
1344
+
1345
+ async def _head(
1346
+ self,
1347
+ path: str,
1348
+ body: bytes,
1349
+ content_type: str,
1350
+ key: str | None,
1351
+ held: str | None,
1352
+ extra: dict,
1353
+ ) -> tuple[int, dict]:
1354
+ """The request sent, over a connection opened where none is kept,
1355
+ and the answer's status and headers."""
1356
+ if self._s.conn is None:
1357
+ self._s.conn = await asyncio.open_connection(
1358
+ self._host, self._port, ssl=self._tls, server_hostname=self._host if self._tls else None
1359
+ )
1360
+ reader, writer = self._s.conn
1361
+ head = [
1362
+ f"POST {self._base}{path} HTTP/1.1",
1363
+ f"Host: {self._host}:{self._port}",
1364
+ f"Content-Type: {content_type}",
1365
+ f"Content-Length: {len(body)}",
1366
+ "Connection: keep-alive",
1367
+ ]
1368
+ if self.token:
1369
+ head.append(f"Authorization: Bearer {self.token}")
1370
+ if key:
1371
+ head.append(f"Idempotency-Key: {key}")
1372
+ if held:
1373
+ head.append(f"Fennec-Wait: {held}")
1374
+ for name, value in extra.items():
1375
+ head.append(f"{name}: {value}")
1376
+ writer.write(("\r\n".join(head) + "\r\n\r\n").encode() + body)
1377
+ await writer.drain()
1378
+
1379
+ status_line = await reader.readuntil(b"\r\n")
1380
+ status = int(status_line.split(b" ", 2)[1])
1381
+ headers: dict[str, str] = {}
1382
+ while (line := await reader.readuntil(b"\r\n")) != b"\r\n":
1383
+ name, _, value = line.decode("latin-1").partition(":")
1384
+ headers[name.strip().lower()] = value.strip()
1385
+ return status, headers
1386
+
1387
+
1388
+ class _Idle:
1389
+ """The clock of an answer's body: `secs` with nothing read, and its
1390
+ connection is let go of (`late`), the read under way meeting its end. A
1391
+ timer the loop holds, where `wait_for` made a task a wait before 3.12."""
1392
+
1393
+ __slots__ = ("secs", "writer", "loop", "handle", "late")
1394
+
1395
+ def __init__(self, writer: asyncio.StreamWriter, secs: float | None):
1396
+ self.secs, self.writer, self.handle, self.late = secs, writer, None, False
1397
+ self.loop = asyncio.get_running_loop()
1398
+
1399
+ def tick(self) -> None:
1400
+ """Starts the clock again: a piece came, or one is to be read."""
1401
+ if self.handle is not None:
1402
+ self.handle.cancel()
1403
+ if self.secs is not None:
1404
+ self.handle = self.loop.call_later(self.secs, self._late)
1405
+
1406
+ def _late(self) -> None:
1407
+ self.late = True
1408
+ self.writer.transport.abort()
1409
+
1410
+ def stop(self) -> None:
1411
+ if self.handle is not None:
1412
+ self.handle.cancel()
1413
+
1414
+
1415
+ async def _exactly(reader: asyncio.StreamReader, n: int, idle: _Idle) -> bytes:
1416
+ """`n` bytes of `reader`, each wait for a piece within `idle`'s clock."""
1417
+ parts = []
1418
+ while n:
1419
+ idle.tick()
1420
+ part = await reader.read(n)
1421
+ if not part:
1422
+ raise asyncio.IncompleteReadError(b"".join(parts), None)
1423
+ parts.append(part)
1424
+ n -= len(part)
1425
+ return parts[0] if len(parts) == 1 else b"".join(parts)
1426
+
1427
+
1428
+ def _gone(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> bool:
1429
+ """Whether a kept stream can no longer take a request: its end read, or
1430
+ an error, as the loop met them. The loop reads a server's close as it
1431
+ comes whenever it waits, as an idle one does; asking the socket as
1432
+ `_closed` does was 1 us a request more, for a loop blocked past the
1433
+ close alone -- whose write without a key then fails, never sent twice."""
1434
+ return reader.at_eof() or reader.exception() is not None or writer.is_closing()
1435
+
1436
+
1437
+ class _AsyncShared(_Shared):
1438
+ """An `AsyncClient`'s connection and what its copies share with it."""
1439
+
1440
+ __slots__ = ("conn", "lock")
1441
+
1442
+ def __init__(self) -> None:
1443
+ super().__init__()
1444
+ self.conn: tuple[asyncio.StreamReader, asyncio.StreamWriter] | None = None
1445
+ self.lock = asyncio.Lock()
1446
+
1447
+
1448
+ def placeholders(start: int, n: int) -> str:
1449
+ """`$start, ..., $(start+n-1)`: a list literal's worth of parameters.
1450
+ FennecQL binds a parameter to a value, not to a list, so `in` takes one
1451
+ per element -- `in [$2, $3, $4]`."""
1452
+ return ", ".join(f"${i}" for i in range(start, start + n))
1453
+
1454
+
1455
+ # Below FennecError, which the builder raises.
1456
+ from .builder import ( # noqa: E402
1457
+ AsyncQuery,
1458
+ Call,
1459
+ Computed,
1460
+ Cond,
1461
+ FacetCount,
1462
+ Param,
1463
+ Query,
1464
+ Rows,
1465
+ _BUILT,
1466
+ _rows,
1467
+ and_,
1468
+ bucket,
1469
+ call,
1470
+ collection,
1471
+ count_distinct,
1472
+ distance,
1473
+ expr,
1474
+ first,
1475
+ inc,
1476
+ last,
1477
+ not_,
1478
+ or_,
1479
+ param,
1480
+ raw,
1481
+ )