fennecdb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fennecdb/relay.py ADDED
@@ -0,0 +1,83 @@
1
+ """Hands every write on a fennec-server's disk to another program or a webhook.
2
+
3
+ python -m fennecdb.relay http://127.0.0.1:8080 --consumer kafka | kcat -P -b broker -t fennec
4
+ python -m fennecdb.relay http://127.0.0.1:8080 --consumer search --to https://indexer/hook
5
+
6
+ The writes are read as a consumer the server keeps the cursor of
7
+ (`GET /_changes?consumer=`, `fennec-server --cdc`), one JSON object a line, and
8
+ the cursor moves only once they are out: flushed to standard output, or
9
+ answered 2xx by the webhook, which is sent them as `application/x-ndjson`
10
+ and sent them again until it takes them. So each write comes at least once
11
+ -- again after a crash between the two -- and whatever takes them should
12
+ be keyed by `seq`, or by the collection and `id`.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import os
20
+ import sys
21
+ import time
22
+ import urllib.error
23
+ import urllib.request
24
+
25
+ from . import Client, FennecError
26
+
27
+
28
+ def deliver(to: str | None, writes: list, timeout: float) -> None:
29
+ body = "".join(json.dumps(w, separators=(",", ":")) + "\n" for w in writes)
30
+ if to is None:
31
+ sys.stdout.write(body)
32
+ sys.stdout.flush()
33
+ return
34
+ req = urllib.request.Request(to, data=body.encode(), method="POST")
35
+ req.add_header("Content-Type", "application/x-ndjson")
36
+ with urllib.request.urlopen(req, timeout=timeout):
37
+ pass
38
+
39
+
40
+ def main(argv: list[str] | None = None) -> int:
41
+ p = argparse.ArgumentParser(prog="python -m fennecdb.relay", description=__doc__.split("\n\n")[0])
42
+ p.add_argument("url", help="the server's HTTP endpoint")
43
+ p.add_argument("--consumer", required=True, help="the name the server keeps the cursor under")
44
+ p.add_argument("--to", help="a webhook to POST each batch to; standard output without")
45
+ p.add_argument("--token", default=os.environ.get("FENNEC_TOKEN"), help="the data token (or FENNEC_TOKEN)")
46
+ p.add_argument("--limit", type=int, default=1000, help="writes a batch holds at most")
47
+ p.add_argument("--once", action="store_true", help="relay what there is, then stop")
48
+ a = p.parse_args(argv)
49
+ db = Client(a.url, token=a.token)
50
+ # Made where there is none; one that is there goes on from where it
51
+ # stands, so a relay stopped and started again misses nothing.
52
+ db.consumer(a.consumer)
53
+ pause = 0.5
54
+ batches = db.follow(a.consumer, limit=a.limit, wait=0 if a.once else 10.0)
55
+ while True:
56
+ try:
57
+ if a.once:
58
+ got = db.changes(consumer=a.consumer, limit=a.limit)
59
+ if not got.writes:
60
+ return 0
61
+ deliver(a.to, got.writes, 30.0)
62
+ db.move_consumer(a.consumer, got.next)
63
+ continue
64
+ writes = next(batches)
65
+ while True:
66
+ try:
67
+ deliver(a.to, writes, 30.0)
68
+ break
69
+ except (OSError, urllib.error.URLError) as e:
70
+ # Not taken: sent again, the cursor where it was.
71
+ print(f"relay: {a.to} did not take the batch ({e}); again in {pause:g} s", file=sys.stderr)
72
+ time.sleep(pause)
73
+ pause = min(pause * 2, 30.0)
74
+ pause = 0.5
75
+ except FennecError as e:
76
+ print(f"relay: {e} ({e.status})", file=sys.stderr)
77
+ return 1
78
+ except BrokenPipeError:
79
+ return 0
80
+
81
+
82
+ if __name__ == "__main__":
83
+ sys.exit(main())
@@ -0,0 +1,301 @@
1
+ Metadata-Version: 2.4
2
+ Name: fennecdb
3
+ Version: 0.2.0
4
+ Summary: fennecdb over HTTP: a client, and vector stores for LangChain and LlamaIndex
5
+ License-Expression: Apache-2.0
6
+ Project-URL: Homepage, https://fennecdb.com
7
+ Project-URL: Documentation, https://fennecdb.com/docs/integrations
8
+ Project-URL: Source, https://github.com/fennec-db/fennecdb
9
+ Keywords: fennecdb,vector,database,langchain,llama-index
10
+ Requires-Python: >=3.10
11
+ Description-Content-Type: text/markdown
12
+ Provides-Extra: langchain
13
+ Requires-Dist: langchain-core>=1.0; extra == "langchain"
14
+ Provides-Extra: llama-index
15
+ Requires-Dist: llama-index-core>=0.12; extra == "llama-index"
16
+ Provides-Extra: test
17
+ Requires-Dist: pytest>=8; extra == "test"
18
+ Requires-Dist: pytest-asyncio>=0.23; extra == "test"
19
+ Requires-Dist: langchain-core>=1.0; extra == "test"
20
+ Requires-Dist: langchain-tests>=1.0; extra == "test"
21
+ Requires-Dist: llama-index-core>=0.12; extra == "test"
22
+
23
+ # fennecdb for Python
24
+
25
+ A client for fennec-server's HTTP endpoint (`fennec-server --http <address>`), and
26
+ vector stores for LangChain and LlamaIndex on top of it.
27
+
28
+ ```sh
29
+ pip install "fennecdb[langchain]" # or "fennecdb[llama-index]"
30
+ ```
31
+
32
+ ```python
33
+ from fennecdb import Client
34
+
35
+ db = Client("http://127.0.0.1:8080", token="...")
36
+ db.query("get articles select title near embed $1 limit 5", [[0.1, 0.2, 0.3]])
37
+ ```
38
+
39
+ Or the query builder writes the statement -- every value a parameter,
40
+ every name checked -- the same text the JavaScript, Go and .NET builders
41
+ make of the same chain:
42
+
43
+ ```python
44
+ from fennecdb import or_
45
+
46
+ docs = db.collection("articles")
47
+ rows = (docs.select("title")
48
+ .where("year", ">=", 2024)
49
+ .where(or_({"lang": "tr"}, {"tags": {"has": "rust"}}))
50
+ .near("embed", [0.1, 0.2, 0.3], ef=64)
51
+ .limit(5)
52
+ .rows())
53
+ docs.insert({"title": "Dunes", "year": 2021})
54
+ docs.where("year", "<", 2000).delete() # no filter: refused unless all=True
55
+ docs.where("lang", "tr").count()
56
+ ```
57
+
58
+ `insert` sends `put`, which writes over a row holding a document's id;
59
+ `insert_new` (`to_insert_new` for the text) sends FennecQL's `insert`,
60
+ refused whole (409, `duplicate`) where one does. `in` takes an inner query
61
+ of one column as well as a list, answered before the query around it, its
62
+ parameters numbered where it stands; `order` takes an expression or a
63
+ `distance` as well as a field:
64
+
65
+ ```python
66
+ from fennecdb import distance, expr
67
+
68
+ docs.insert_new({"id": 7, "title": "Oasis"}, returning=["id"], require=1)
69
+ orders = db.collection("orders")
70
+ in_tr = db.collection("customers").select("id").where("country", "TR")
71
+ orders.where("customer", "in", in_tr).rows()
72
+ orders.where({"customer": {"in": in_tr}}).count()
73
+ orders.order(expr("n * 2"), "desc").order("item").limit(10).rows()
74
+ db.collection("places").order(distance("loc", [13.4, 52.5])).limit(5).rows()
75
+ ```
76
+
77
+ Aggregates go in `select` as FennecQL spells them, and `group` takes one key
78
+ or more; `bucket`, `count_distinct`, `first`, `last` and `expr` make a
79
+ column, named with `.as_(name)` (`as` being Python's keyword):
80
+
81
+ ```python
82
+ from fennecdb import bucket, expr, first, last
83
+
84
+ bars = (db.collection("ticks")
85
+ .select(bucket("at", "1m").as_("minute"), first("px", "at").as_("open"),
86
+ "max(px)", "min(px)", last("px", "at").as_("close"),
87
+ expr("sum(px * qty) / sum(qty)").as_("vwap"))
88
+ .where("sym", "S07")
89
+ .group("minute")
90
+ .order("minute", "desc")
91
+ .rows())
92
+ ```
93
+
94
+ `facet(field, ranges=[0, 25, 50])` counts by ranges of numbers, each
95
+ value `[from, to]`, and `disjunctive=True` counts past the filter's own
96
+ condition on the field, as a shop's filter list does.
97
+
98
+ Several statements are one `batch`, a block under one write lock: all
99
+ land or none, and a `FennecError` that stops it says which (`e.at`, from 0;
100
+ 412 for a write whose `require` was not met). `reason=` beside `require=`
101
+ (`require 1 else 'insufficient funds'`, written into the text) comes back
102
+ as `e.reason`, which tells one guard's refusal from another's where `e.at`
103
+ moves with every statement added in front; a read's `.require(1,
104
+ reason="no such account")` says its own. On a tenant node a quota
105
+ names itself (`e.quota`): `"bytes"` and `"rows"` answer 507, and room is
106
+ made by deleting; `"streams"`, `"requests"`, `"writes"` and `"write_bytes"`
107
+ answer 429 with `e.retry_after`, the seconds the server asks to wait, as a
108
+ tenant being moved or failed over answers its writes 503. A retry after a timeout
109
+ cannot tell whether a write ran, so a write that must not run twice goes
110
+ under an idempotency key -- the second answer is the first one kept
111
+ (`db.replayed`), and the key with another request is refused (422):
112
+
113
+ ```python
114
+ from fennecdb import inc
115
+
116
+ debit = docs.where("id", 1).to_update({"stock": inc(-1)}, require=1)
117
+ db.batch([debit, ("put orders {item: $1}", [1])], idempotency_key=order_id)
118
+ db.with_idempotency_key(order_id).collection("orders").insert({"item": 1})
119
+ db.seq # the change the last write left the database at
120
+ ```
121
+
122
+ The client makes that retry itself where it is safe: a request the server
123
+ says ran nothing -- a 503, or a 429 naming its wait -- goes again after that
124
+ wait and never sooner, as does one whose connection was refused; a read the
125
+ builder made or sent near, or a write under a key, goes again after a
126
+ dropped connection too. A refusal naming no wait never does, nor a write
127
+ without a key once it may have landed, nor a text given `read=True` and
128
+ sent home, a claim nothing checks. `timeout` (seconds, 30 unless given)
129
+ bounds the wait for an answer's head and between the pieces of its body,
130
+ and nothing goes again after it, since the server may still be running the
131
+ request: give a `compact` or a `create index` a client with a longer one.
132
+
133
+ ```python
134
+ from fennecdb import Client, Retry
135
+
136
+ db = Client(url, timeout=10, retry=Retry(attempts=5, max_wait=30))
137
+ db = Client(url, retry=False) # one try, whatever the answer
138
+ maintenance = Client(url, timeout=600) # a compact runs long before it answers
139
+ ```
140
+
141
+ One value in two places is one parameter with `param`, so the amount a
142
+ write takes and the amount its filter checks cannot drift apart: each use
143
+ of the same `param` writes the same `$n`, where two equal values are two:
144
+
145
+ ```python
146
+ from fennecdb import expr, param
147
+
148
+ amount = param(300)
149
+ (db.collection("accounts").where({"name": "ada", "balance": {"gte": amount}})
150
+ .update({"balance": expr("balance - ?", amount)}, require=1))
151
+ # set accounts {balance: balance - $1} where name = $2 and balance >= $1 require 1
152
+ ```
153
+
154
+ A param goes where a whole value stands: inside a list or a json value it
155
+ is refused. `expr`'s text, and `raw`'s, is trusted FennecQL, written into the
156
+ statement as it is: the code's own, never a value from a user, which goes in
157
+ as a `?`'s parameter.
158
+
159
+
160
+ A job queue's claim is one statement: `order` and `limit` before
161
+ `update` pick the rows it writes, and `returning=True` (or a list of
162
+ fields) answers them as written, where a count is answered otherwise:
163
+
164
+ ```python
165
+ from fennecdb import expr, inc, raw
166
+
167
+ claimed = (
168
+ db.collection("jobs").where(raw("run_at <= now()")).order("run_at").limit(10)
169
+ .update({"owner": me, "run_at": expr("now() + ?", 30000), "attempts": inc(1)}, returning=True, wait=30)
170
+ )
171
+ ```
172
+
173
+ With `wait` (seconds, at most 30) a claim that finds no job is held at
174
+ the server until one comes -- enqueued, a delayed one due, a lease
175
+ lapsed -- rather than sent again every few: answered with the jobs it
176
+ took, or with none at the end of the wait. `query` and `batch` take it
177
+ too, and `AsyncClient`'s give a held request up when cancelled.
178
+
179
+ A named statement (`create statement transfer(...) as ...`) is called by
180
+ name, its arguments a mapping of values by name -- one parameter, `call
181
+ transfer $1`, sent over `/query` as any statement is, so the errors, the
182
+ key and `wait` are a statement's own:
183
+
184
+ ```python
185
+ from fennecdb import call
186
+
187
+ db.call("transfer", {"from": 1, "to": 2, "amount": 300}, idempotency_key=transfer_id)
188
+ db.call("stamp") # no arguments: call stamp
189
+ db.call("mine", claims={"sub": "alice"}) # call mine as $1
190
+ db.call("history", {"limit": 10}, read=True) # goes near on reads="nearest"
191
+ await async_db.call("transfer", {"from": 1, "to": 2, "amount": 300})
192
+ text, params = call("transfer", {"from": 1, "to": 2, "amount": 300}).to_call()
193
+ db.batch([(text, params), debit]) # a call inside a batch
194
+ ```
195
+
196
+ `db.call(name, args=None, *, claims=None, idempotency_key=None, wait=None,
197
+ read=False)` answers as `query` does -- rows, or the affected count for a
198
+ body that answers none -- and `AsyncClient.call` is the same awaited. The
199
+ arguments and the claims are parameters, normalised as a document's values
200
+ are (a `datetime` its ISO text, an `array.array` its numbers); only the
201
+ name is written into the text, so it is checked: ASCII letters, digits and
202
+ `_`, a letter first, 59 bytes at most, else a `FennecError` of status 0
203
+ (`invalid statement name: ...`), as is arguments or claims that are not a
204
+ mapping (`a call's arguments are an object of values by name`). `claims`
205
+ is the `as` part, the server's token alone may say it (a scoped token's
206
+ are its own). `read=True` says the statement only reads: on a client with
207
+ `reads="nearest"` it is answered near the client, after its writes, and a
208
+ statement that writes is refused there (403), never landed; not said, a
209
+ call goes home. Errors are `FennecError`'s, by status: 404 no such
210
+ statement or none granted, 403 a claim missing, 400 an argument missing,
211
+ extra or of another type, 409 a unique value taken, 412 a `require` not met
212
+ with its `e.reason`.
213
+
214
+ `db.statements()` -- awaited on `AsyncClient` -- lists the statements the
215
+ server holds, as it lists them to its own token (a scoped one is refused,
216
+ 403): each a `NamedStatement` with its `signature`, `writes`, `comment`,
217
+ `broken`, a binding's `after`, and a scheduled statement's clock -- its
218
+ `schedule` (`every 1h`, `at '0 3 * * *' in 'Europe/Berlin'`), the `next`
219
+ slot as an aware UTC `datetime`, the `last` run (a `LastRun`: `slot`,
220
+ `outcome` `"ran"` or `"refused"`, `status`, `reason`, `skipped`) and a slot
221
+ the server holds back (`held`, a `HeldSlot`); and a webhook consumer's `hook`
222
+ and whether the server holds its secret (`armed`: `False` answers every
223
+ event 401). `db.call` of a scheduled one runs it now, as the server's token,
224
+ marking no slot; of a webhook consumer, `db.call("stripe_paid", event)`
225
+ sends it a test event, run as a delivery is, keeping no key.
226
+
227
+ To see the text and parameters a chain builds, for logging or a test,
228
+ `to_fennecql()` returns them and runs nothing:
229
+ `docs.select("title").limit(5).to_fennecql()` is
230
+ `("get articles select title limit 5", [])`. A query needs no call to it
231
+ before it runs.
232
+
233
+ In an event loop -- FastAPI, aiohttp, an agent -- `AsyncClient` makes the
234
+ same calls awaited, over one kept-alive connection, the standard library
235
+ alone:
236
+
237
+ ```python
238
+ from fennecdb import AsyncClient
239
+
240
+ async with AsyncClient("http://127.0.0.1:8080", token="...") as db:
241
+ rows = await db.query("get articles select title near embed $1 limit 5", [[0.1, 0.2, 0.3]])
242
+ await db.batch([("put articles {title: $1}", ["a"]), ("put articles {title: $1}", ["b"])])
243
+ rows = await db.collection("articles").where("year", ">=", 2024).rows()
244
+ ```
245
+
246
+ Every write on the server's disk (`fennec-server --cdc`), for a consumer whose
247
+ cursor the server keeps -- each batch committed once the loop comes back
248
+ for the next, so each write comes at least once, and a loop started again
249
+ goes on from the last batch committed -- and the same handed on to another
250
+ program or a webhook. `db.consumer(name)` makes one where there is none and
251
+ leaves one that is there where it stands; `db.move_consumer(name, since)`
252
+ moves it:
253
+
254
+ ```python
255
+ for batch in db.follow("search-index"):
256
+ for change in batch: # {"seq", "at", "collection", "op", "id", "doc"}
257
+ index(change)
258
+ ```
259
+
260
+ ```sh
261
+ python -m fennecdb.relay http://127.0.0.1:8080 --consumer kafka | kcat -P -b broker:9092 -t fennec
262
+ ```
263
+
264
+ ```python
265
+ from fennecdb.langchain import FennecVectorStore
266
+
267
+ store = FennecVectorStore(embeddings, "docs", url="http://127.0.0.1:8080", token="...",
268
+ metadata_fields={"source": "text"})
269
+ store.add_documents(documents)
270
+ store.similarity_search("how do I compact", k=4, filter={"source": "handbook"})
271
+ ```
272
+
273
+ ```python
274
+ from fennecdb.llama_index import FennecVectorStore
275
+ from llama_index.core import StorageContext, VectorStoreIndex
276
+
277
+ store = FennecVectorStore("docs", url="http://127.0.0.1:8080", token="...")
278
+ index = VectorStoreIndex.from_documents(
279
+ documents, storage_context=StorageContext.from_defaults(vector_store=store)
280
+ )
281
+ ```
282
+
283
+ A document or node is a row -- its id, text, metadata as JSON and vector
284
+ under `@hnsw` -- in a collection created on the first write. Fields named in
285
+ `metadata_fields` get columns of their own under `@hash`, and filters over
286
+ them are answered by the index before the search runs.
287
+
288
+ With `full_text=True` the text is indexed for BM25 too, and both stores search
289
+ by the words alone or by the words and the vector fused -- LangChain's
290
+ `mode="text"` and `mode="hybrid"`, LlamaIndex's `TEXT_SEARCH` and `HYBRID`.
291
+ `quant="int8"` or `"bit"` keeps the graph over codes.
292
+
293
+ ```python
294
+ store = FennecVectorStore(embeddings, "docs", url=..., token=..., full_text=True)
295
+ store.similarity_search("how do I compact", k=4, mode="hybrid")
296
+ ```
297
+
298
+ `./run-tests.sh` runs LangChain's standard vector store suite and the tests
299
+ LlamaIndex's own integrations run against a fennec-server it builds and starts,
300
+ and the query builder against every case of `integrations/builder-golden.json`.
301
+ Full reference: https://fennecdb.com/docs/integrations
@@ -0,0 +1,9 @@
1
+ fennecdb/__init__.py,sha256=19EHmIItaHgT62d5NC_pUI1v_0aaHShDuPVeLVkubcg,61616
2
+ fennecdb/builder.py,sha256=48i1PP5TzqqdxybIaxBcZ5bN3gqEnhhbJoYK0snKnQE,76922
3
+ fennecdb/langchain.py,sha256=AsAYriS7_K9Wfm8o5aw-DY8L5AUnKFcaPxwm_4LnTac,12660
4
+ fennecdb/llama_index.py,sha256=aBo3dJqqhxheGGDxzQHa2aDHpWsuL9SKgKqs1BiQhMk,13643
5
+ fennecdb/relay.py,sha256=55vKjlG0UB8iNt4DxbldQV7NR9djHof8Tscq4K7Qjy4,3448
6
+ fennecdb-0.2.0.dist-info/METADATA,sha256=Vd1-XklyQvGGhBqfsC_l-A2_vTgOqDusYyyHuoNkGuY,13747
7
+ fennecdb-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ fennecdb-0.2.0.dist-info/top_level.txt,sha256=2TolaoNqFjFSgB0ZPToGKdjjg7NyPnniBWzjvEZwmGA,9
9
+ fennecdb-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ fennecdb