semurg 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
semurg/__init__.py ADDED
@@ -0,0 +1,43 @@
1
+ """Semurg Python client.
2
+
3
+ Connect to your own self-hosted Semurg node over HTTP (working today), WebSocket (interface,
4
+ roadmap wire) or Semurg ATP (frame codec live, wire roadmap).
5
+
6
+ from semurg import Client
7
+ c = Client("http://127.0.0.1:4000", token="...")
8
+ print(c.health())
9
+ """
10
+
11
+ from . import frame
12
+ from .client import Client, __version__
13
+ from .errors import (
14
+ SemurgAuthError,
15
+ SemurgConnectionError,
16
+ SemurgError,
17
+ SemurgHTTPError,
18
+ SemurgNotFound,
19
+ SemurgRateLimited,
20
+ SemurgRoadmapError,
21
+ SemurgServerError,
22
+ SemurgTimeout,
23
+ )
24
+ from .transports import AtpTransport, HttpTransport, Transport, WsTransport
25
+
26
+ __all__ = [
27
+ "Client",
28
+ "__version__",
29
+ "frame",
30
+ "Transport",
31
+ "HttpTransport",
32
+ "WsTransport",
33
+ "AtpTransport",
34
+ "SemurgError",
35
+ "SemurgHTTPError",
36
+ "SemurgAuthError",
37
+ "SemurgNotFound",
38
+ "SemurgRateLimited",
39
+ "SemurgServerError",
40
+ "SemurgTimeout",
41
+ "SemurgConnectionError",
42
+ "SemurgRoadmapError",
43
+ ]
semurg/client.py ADDED
@@ -0,0 +1,331 @@
1
+ """The Semurg client.
2
+
3
+ Point it at your OWN Semurg node (a self-hosted instance on your own hardware) and call the
4
+ same methods you would against any Semurg box. HTTP is the working transport today; ``ws`` and
5
+ ``atp`` are pluggable interfaces whose live wires are on the roadmap (see clients/CONNECTING.md).
6
+
7
+ from semurg import Client
8
+
9
+ with Client("http://127.0.0.1:4000") as c:
10
+ print(c.health())
11
+ print(c.query("search", {"needle": "hello"}))
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import base64
17
+ import struct
18
+ from typing import Any, Optional
19
+ from urllib.parse import quote
20
+
21
+ from . import frame as _frame
22
+ from .errors import SemurgError
23
+ from .transports import Transport, build_transport
24
+
25
+ __version__ = "0.1.0"
26
+
27
+
28
+ class Client:
29
+ """A Semurg node client with a pluggable transport.
30
+
31
+ Args:
32
+ base_url: your node's base URL, for example ``http://127.0.0.1:4000`` or
33
+ ``https://one.semurg.io``. The SDK appends ``/api/...`` and ``/v1/...``.
34
+ token: optional API token, sent as ``Authorization: Bearer <token>``. A node that has
35
+ no token provisioned is open (read-only public build); a node with a token rejects
36
+ unauthenticated ``/v1`` calls with 401.
37
+ timeout: per-request timeout in seconds.
38
+ transport: ``"http"`` (default, working), ``"ws"`` or ``"atp"`` (interfaces, roadmap wire).
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ base_url: str,
44
+ token: Optional[str] = None,
45
+ timeout: float = 30.0,
46
+ transport: str = "http",
47
+ ) -> None:
48
+ self.base_url = base_url.rstrip("/")
49
+ self.token = token
50
+ self.timeout = timeout
51
+ self.transport: Transport = build_transport(transport, self.base_url, token, timeout)
52
+
53
+ # -- lifecycle ---------------------------------------------------------------------------
54
+
55
+ def close(self) -> None:
56
+ self.transport.close()
57
+
58
+ def __enter__(self) -> "Client":
59
+ return self
60
+
61
+ def __exit__(self, *exc: Any) -> None:
62
+ self.close()
63
+
64
+ # -- benchmark / meta API (/api) ---------------------------------------------------------
65
+
66
+ def health(self) -> dict:
67
+ """GET /api/health -- liveness, engine state, dataset readiness."""
68
+ return self.transport.request("GET", "/api/health")
69
+
70
+ def version(self) -> dict:
71
+ """Client + server identity. Semurg exposes no dedicated version route; this summarizes
72
+ /api/health (service) alongside the SDK version."""
73
+ h = self.health()
74
+ return {
75
+ "client": __version__,
76
+ "service": h.get("service") if isinstance(h, dict) else None,
77
+ "ok": h.get("ok") if isinstance(h, dict) else None,
78
+ }
79
+
80
+ # -- data plane (/v1) --------------------------------------------------------------------
81
+
82
+ def status(self) -> dict:
83
+ """GET /v1/status -- instance identity, licence, writability, store state."""
84
+ return self.transport.request("GET", "/v1/status")
85
+
86
+ # -- whole-cluster (five surfaces, one capability) --------------------------------------------
87
+
88
+ def cluster_util(self, window_ms: Optional[int] = None) -> dict:
89
+ """GET /v1/cluster/util -- live whole-cluster utilisation (per-pipe GB/s, per-core %, memBW)."""
90
+ path = "/v1/cluster/util"
91
+ if window_ms is not None:
92
+ path += "?window_ms=" + str(int(window_ms))
93
+ return self.transport.request("GET", path)
94
+
95
+ def conveyor_report(self) -> dict:
96
+ """GET /v1/cluster/conveyor -- the node's process-global conveyor counters: the belt
97
+ (odirect_open_ok, odirect_open_fallback, buffered_preads, sqes_submitted, uring_completions,
98
+ fifo_*, hugepage_*), the graph prefetch (prefetch_sqes_ahead, prefetch_used,
99
+ prefetch_evicted_unused) and the janitor (janitor_epochs, relocated_containers, pages_before,
100
+ pages_after) witnesses. Returns {ok, conveyor}; operator detail, so an anonymous caller gets
101
+ conveyor = {disclosed: false}. The same map the authenticated /api/health carries under
102
+ conveyor.counters and the ATP CONVEYOR_REPORT op (10) returns."""
103
+ return self.transport.request("GET", "/v1/cluster/conveyor")
104
+
105
+ def licence(self) -> dict:
106
+ """GET /v1/licence -- the full licence status (licensee, cluster?, node_count, used, headroom)."""
107
+ return self.transport.request("GET", "/v1/licence")
108
+
109
+ def nodes(self) -> dict:
110
+ """GET /v1/cluster/nodes -- the whole-cluster membership."""
111
+ return self.transport.request("GET", "/v1/cluster/nodes")
112
+
113
+ def node_status(self, fp: str) -> dict:
114
+ """GET /v1/cluster/nodes/<fp> -- one node's status by fingerprint."""
115
+ return self.transport.request("GET", "/v1/cluster/nodes/" + str(fp))
116
+
117
+ def query(self, op: str, args: Optional[dict] = None, **kwargs: Any) -> dict:
118
+ """POST /v1/query -- batch read. ``op`` is fetch | resolve | search | histogram."""
119
+ body = {"op": op}
120
+ if args:
121
+ body.update(args)
122
+ if kwargs:
123
+ body.update(kwargs)
124
+ return self.transport.request("POST", "/v1/query", json_body=body)
125
+
126
+ def fetch(self, ids: list[int]) -> dict:
127
+ """/v1/query op=fetch -- resolve a batch of container ids (base64 containers in the result)."""
128
+ return self.query("fetch", {"ids": list(ids)})
129
+
130
+ def resolve(self, id: int) -> dict:
131
+ """/v1/query op=resolve -- resolve a single container id."""
132
+ return self.query("resolve", {"id": id})
133
+
134
+ def search(self, needle: str) -> dict:
135
+ """/v1/query op=search -- index-free pattern scan for ``needle``."""
136
+ return self.query("search", {"needle": needle})
137
+
138
+ def histogram(self, field_offset: int = 8, buckets: int = 16) -> dict:
139
+ """/v1/query op=histogram -- one-pass histogram over a byte field (offset 0..63)."""
140
+ return self.query("histogram", {"field_offset": field_offset, "buckets": buckets})
141
+
142
+ def vector_top_k(self, queries: list[list[float]], k: int = 10, ef: Optional[int] = None) -> dict:
143
+ """/v1/query op=vector_top_k -- exact nearest neighbours for a batch of float vectors.
144
+
145
+ ``queries`` is a list of equal-length vectors. They travel as one row-major little-endian
146
+ float32 matrix, base64 encoded, because JSON carries no raw bytes and a second encoding is a
147
+ second thing for the two ends to disagree about. The answer is one list per query, in input
148
+ order, each ``[distance, id]`` ascending. The distance is squared, so it is comparable and not
149
+ a metre; take its square root if a reader needs one.
150
+ """
151
+ import base64
152
+ import struct
153
+
154
+ if not queries:
155
+ raise ValueError("vector_top_k needs at least one query")
156
+ d = len(queries[0])
157
+ if any(len(q) != d for q in queries):
158
+ raise ValueError("every query must have the same number of dimensions")
159
+ flat = [float(x) for q in queries for x in q]
160
+ raw = struct.pack("<%df" % len(flat), *flat)
161
+ body = {"queries_b64": base64.b64encode(raw).decode("ascii"), "q": len(queries), "d": d, "k": k}
162
+ # ``ef`` (V-INDEX, r11.47+) asks for the INDEXED answer: a best-first search of the store's k-NN graph on disk
163
+ # with ``ef`` candidates per shard, every distance still exact; the reply says which board answered ("answer")
164
+ # and the ef the node applied ("ef"). A node that predates the index answers exact with no "answer" key: that
165
+ # reply is refused by name, never returned as an indexed one.
166
+ if ef is not None:
167
+ body["ef"] = ef
168
+ res = self.query("vector_top_k", body)
169
+ if ef is not None and not (isinstance(res, dict) and res.get("answer") == "indexed"):
170
+ raise SemurgError(
171
+ "the node answered exact: it predates the k-NN index (ef needs r11.47+); drop ef or upgrade the node"
172
+ )
173
+ return res
174
+
175
+ def vector_search(self, query: str, k: int = 10) -> dict:
176
+ """/v1/query op=vector_search -- exact top-k over the in-store SimHash codes of ``query``'s embedding.
177
+
178
+ The node embeds ``query`` with its own model, so the codes compare with the passages written through
179
+ ``ingest(passages=...)``. Rows are ``{id, hamming, similarity, passage}``; the envelope's ``note`` states the
180
+ recall framing (recall@1 exact, recall@k a quantisation of float-cosine NN)."""
181
+ return self.query("vector_search", {"query": query, "k": k})
182
+
183
+ def similar(self, id: int, k: int = 10) -> dict:
184
+ """/v1/query op=similar -- the ``k`` nodes nearest to stored node ``id`` by its own code (no re-embed)."""
185
+ return self.query("similar", {"id": id, "k": k})
186
+
187
+ def ingest_vectors(self, vectors: list[list[float]], ids: list[int]) -> dict:
188
+ """POST /v1/ingest ``vectors_b64`` -- store float vectors under your own ids (a WRITE).
189
+
190
+ ``vectors`` are equal-length (1..176 components), ``ids`` one per vector, distinct, ``1 <= id < 2**60``.
191
+ They travel as one row-major little-endian float32 matrix, base64 encoded, the same encoding
192
+ ``vector_top_k`` sends its queries in, and ``vector_top_k`` answers with these ids. One store holds one
193
+ vector width; a query of another width is refused by name."""
194
+ if not vectors:
195
+ raise ValueError("ingest_vectors needs at least one vector")
196
+ d = len(vectors[0])
197
+ if any(len(v) != d for v in vectors):
198
+ raise ValueError("every vector must have the same number of components")
199
+ if len(ids) != len(vectors):
200
+ raise ValueError("one id per vector")
201
+ flat = [float(x) for v in vectors for x in v]
202
+ raw = struct.pack("<%df" % len(flat), *flat)
203
+ body = {"vectors_b64": base64.b64encode(raw).decode("ascii"), "d": d, "ids": [int(i) for i in ids]}
204
+ return self.transport.request("POST", "/v1/ingest", json_body=body)
205
+
206
+ def fetch_batch(self, ids: list[int]) -> list[bytes]:
207
+ """POST /v1/query {"op":"fetch"} with Accept: application/octet-stream -- the BINARY fast path
208
+ (fetch has one door; the encoding is the Accept header). Returns one frame per id in request
209
+ order (a zero-filled frame for an absent id)."""
210
+ body = {"op": "fetch", "ids": [int(i) & 0xFFFFFFFFFFFFFFFF for i in ids]}
211
+ blob = self.transport.request("POST", "/v1/query", json_body=body, accept="bin")
212
+ return _frame.unpack(blob)
213
+
214
+ # -- memory (/v1/memory): the AI memory verbs, the same bodies the REST routes take -------
215
+
216
+ def memory_status(self, user_id: Optional[str] = None) -> dict:
217
+ """GET /v1/memory/status -- is the capability on, and what scopes exist. With ``user_id`` (an end-user
218
+ partition inside your identity) it lists that partition's scopes."""
219
+ path = "/v1/memory/status"
220
+ if user_id is not None:
221
+ path += "?user_id=" + quote(user_id, safe="")
222
+ return self.transport.request("GET", path)
223
+
224
+ def remember(
225
+ self,
226
+ scope: str,
227
+ content: str | None = None,
228
+ facts: dict | None = None,
229
+ passages: list | None = None,
230
+ user_id: str | None = None,
231
+ **kw: Any,
232
+ ) -> dict:
233
+ """POST /v1/memory/remember -- store pre-extracted facts and/or passages in ``scope`` (a WRITE).
234
+
235
+ ``user_id`` (optional) names an END-USER partition inside your identity: the memory is kept apart per
236
+ user_id and ``delete_user`` removes it whole. The node validates it (no ``:``, at most 256 bytes) and
237
+ refuses a bad one by name; the SDK sends it as is.
238
+
239
+ ``facts`` takes either the full shape ``{"entities": [...], "relations": [...]}`` or the short
240
+ form ``{"subject": "Paris", "kind": "city"}``. A facts map that names nothing storable is
241
+ refused by the node rather than acknowledged.
242
+
243
+ ``passages`` stores text now, with no model: ``[{"text": "Lena works at Acme.", "mentions":
244
+ ["Lena", "Acme"], "key": "memo-1"}]`` (``key`` optional). Each mention must appear in its text
245
+ (letter case aside), or the node refuses the passage by name. ``memory_context`` returns the
246
+ passage whole when it reaches a mentioned entity. Raw ``content`` needs a text extractor the
247
+ node may not have."""
248
+ body: dict = {"scope": scope, **kw}
249
+ if content is not None:
250
+ body["content"] = content
251
+ if facts is not None:
252
+ body["facts"] = facts
253
+ if passages is not None:
254
+ body["passages"] = passages
255
+ if user_id is not None:
256
+ body["user_id"] = user_id
257
+ return self.transport.request("POST", "/v1/memory/remember", json_body=body)
258
+
259
+ def recall(self, scope: str, subject: str | None = None, query: str | None = None, k: int | None = None,
260
+ user_id: str | None = None) -> dict:
261
+ """POST /v1/memory/recall -- walk outward from an exact entity and answer WITH provenance. ``user_id``
262
+ (optional) reads that end-user partition only."""
263
+ body: dict = {"scope": scope}
264
+ if user_id is not None:
265
+ body["user_id"] = user_id
266
+ if subject is not None:
267
+ body["subject"] = subject
268
+ if query is not None:
269
+ body["query"] = query
270
+ if k is not None:
271
+ body["k"] = int(k)
272
+ return self.transport.request("POST", "/v1/memory/recall", json_body=body)
273
+
274
+ def context(
275
+ self, scope: str, query: str, k: int | None = None, hops: int | None = None, user_id: str | None = None
276
+ ) -> dict:
277
+ """POST /v1/memory/context -- the ranked, cited facts of a scope relevant to a question (GraphRAG retrieval; no model is called). ``user_id`` (optional) reads that end-user partition only."""
278
+ body: dict = {"scope": scope, "query": query}
279
+ if user_id is not None:
280
+ body["user_id"] = user_id
281
+ if k is not None:
282
+ body["k"] = int(k)
283
+ if hops is not None:
284
+ body["hops"] = int(hops)
285
+ return self.transport.request("POST", "/v1/memory/context", json_body=body)
286
+
287
+ def relate(self, scope: str, frm: str, to: str, type: str = "RELATED_TO", user_id: str | None = None) -> dict:
288
+ """POST /v1/memory/relate -- one typed edge between two entities in ``scope`` (a WRITE). ``user_id``
289
+ (optional) writes into that end-user partition."""
290
+ body: dict = {"scope": scope, "from": frm, "to": to, "type": type}
291
+ if user_id is not None:
292
+ body["user_id"] = user_id
293
+ return self.transport.request("POST", "/v1/memory/relate", json_body=body)
294
+
295
+ def forget(self, scope: str, subject: str, reason: str | None = None, user_id: str | None = None) -> dict:
296
+ """POST /v1/memory/forget -- supersede an entity in ``scope`` (a WRITE; nothing is erased). ``user_id``
297
+ (optional) forgets inside that end-user partition."""
298
+ body: dict = {"scope": scope, "subject": subject}
299
+ if reason is not None:
300
+ body["reason"] = reason
301
+ if user_id is not None:
302
+ body["user_id"] = user_id
303
+ return self.transport.request("POST", "/v1/memory/forget", json_body=body)
304
+
305
+ def delete_user(self, user_id: str) -> dict:
306
+ """POST /v1/memory/delete_user -- delete one END USER's whole memory partition (a WRITE; the owner's right to erasure).
307
+
308
+ The reply says what left: ``state`` (``"deleted"`` or ``"already_deleted"``), ``scopes``, ``entities``,
309
+ ``documents``, ``tombstoned_nodes``, ``tombstoned_edges``, ``reads``, ``written_during_delete``,
310
+ ``keyword_postings``. A user_id the node does not know is a 404 (``SemurgError``), a missing one 422, and a
311
+ public sandbox refuses it (403)."""
312
+ return self.transport.request("POST", "/v1/memory/delete_user", json_body={"user_id": user_id})
313
+
314
+ def ingest(self, containers: Any, stamp_edges: bool = False) -> dict:
315
+ """POST /v1/ingest -- append a batch of frames (needs the write capability).
316
+
317
+ ``containers`` is either a single packed ``bytes`` (a whole multiple of the frame size)
318
+ or a list of frames (as returned by :func:`semurg.frame.encode`). ``stamp_edges=True`` asks the
319
+ node to write the content address of every edge sent with its id left at 0 (a node that does not
320
+ know the flag refuses such an edge by name, so nothing is stored wrong)."""
321
+ if isinstance(containers, (bytes, bytearray)):
322
+ packed = bytes(containers)
323
+ else:
324
+ packed = _frame.pack(list(containers))
325
+ if not packed or len(packed) % _frame.CONTAINER_BYTES != 0:
326
+ raise ValueError("ingest expects a whole number of frames")
327
+ b64 = base64.b64encode(packed).decode("ascii")
328
+ body: dict = {"containers_b64": b64}
329
+ if stamp_edges:
330
+ body["stamp_edges"] = True
331
+ return self.transport.request("POST", "/v1/ingest", json_body=body)
semurg/errors.py ADDED
@@ -0,0 +1,66 @@
1
+ """Typed errors for the Semurg client.
2
+
3
+ Every failure the SDK raises is a subclass of :class:`SemurgError`, so a caller can
4
+ catch one thing and still branch on the specific case. HTTP status codes map to the
5
+ subclasses below exactly as the Semurg node returns them (see clients/CONNECTING.md).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, Optional
11
+
12
+
13
+ class SemurgError(Exception):
14
+ """Base class for every error this client raises."""
15
+
16
+
17
+ class SemurgTimeout(SemurgError):
18
+ """The request did not complete within the client timeout."""
19
+
20
+
21
+ class SemurgConnectionError(SemurgError):
22
+ """The node could not be reached at all (DNS, refused, TLS, reset)."""
23
+
24
+
25
+ class SemurgHTTPError(SemurgError):
26
+ """A non-2xx HTTP response. Carries the status and the decoded body when present."""
27
+
28
+ def __init__(self, status: int, message: str, body: Any = None) -> None:
29
+ self.status = status
30
+ self.message = message
31
+ self.body = body
32
+ super().__init__(f"HTTP {status}: {message}")
33
+
34
+
35
+ class SemurgAuthError(SemurgHTTPError):
36
+ """401 (token required / wrong token) or 403 (instance is read-only / missing capability)."""
37
+
38
+
39
+ class SemurgNotFound(SemurgHTTPError):
40
+ """404 (unknown benchmark key, or route not present on this node)."""
41
+
42
+
43
+ class SemurgRateLimited(SemurgHTTPError):
44
+ """429 (rate/concurrency limit) or a 503 warming response, both carrying Retry-After.
45
+
46
+ ``retry_after`` is the number of seconds the node asks the caller to wait, parsed from
47
+ the ``Retry-After`` header (falling back to a ``retry_after`` field in the JSON body).
48
+ """
49
+
50
+ def __init__(self, status: int, message: str, retry_after: Optional[float] = None, body: Any = None) -> None:
51
+ self.retry_after = retry_after
52
+ super().__init__(status, message, body)
53
+
54
+
55
+ class SemurgServerError(SemurgHTTPError):
56
+ """5xx other than a warming 503 (for example a 500 from a failed append)."""
57
+
58
+
59
+ class SemurgRoadmapError(SemurgError):
60
+ """The selected transport's live wire is not shipped yet.
61
+
62
+ Raised by the WebSocket and ATP transports for request/response operations whose
63
+ external wire is on the roadmap. The message names the working alternative (HTTP)
64
+ and the real functionality the transport still provides (frame codec, message
65
+ builders). The SDK never fakes a live wire.
66
+ """
semurg/frame.py ADDED
@@ -0,0 +1,187 @@
1
+ """Semurg ATP Overdrive-64 frame codec.
2
+
3
+ A frame is the fixed-size record the Semurg wire moves (see the byte layout below). This module
4
+ encodes and decodes it in pure Python, byte-for-byte identical to the node's frame codec (same
5
+ CRC-32C, same little-endian field layout), so a frame you build here is the exact wire unit the
6
+ ATP transport will send once the external ATP wire ships (see clients/CONNECTING.md, ATP is
7
+ ROADMAP for the live wire; this codec is live today and unit-tested offline).
8
+
9
+ Header layout (little-endian):
10
+
11
+ byte 0 type (high nibble) | flags (low nibble)
12
+ byte 1 flags_ext / nonce-lo
13
+ bytes 2..5 generation (u32, ABA-safe handle stamp)
14
+ bytes 6..7 schema (u16)
15
+ bytes 8..15 logical_id (u64) -- Node ENTITY-ID / Edge SRC-ID / Chunk CHUNK-ID
16
+ bytes 16..23 edge_dst_id (u64) -- meaningful for edges (type 0x2)
17
+ bytes 24..59 body (36 bytes, form-specific)
18
+ bytes 60..63 CRC-32C over bytes 0..59
19
+
20
+ ID performance: keep ``logical_id`` <= ``DENSE_ID_MAX`` (~4.29e9) for the engine's O(1) arithmetic
21
+ id-locate -- flat throughput and flat memory at any scale. A larger (sparse) id spills into a
22
+ disk-resident extendible hash: still correct, but with a gentler per-insert decay as the index grows.
23
+ Use :func:`dense_ids` (or :func:`dense_id` for multi-model) to assign bulk unique keys; do NOT high-bit-pack
24
+ a model prefix (e.g. ``model_id<<40``), which forces EVERY id onto the slower sparse arm.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from dataclasses import dataclass
30
+
31
+ CONTAINER_BYTES = 64
32
+ BODY_OFFSET = 24
33
+ BODY_MAX = 60 - BODY_OFFSET # 36 bytes free before the CRC
34
+
35
+ TYPE_TOMBSTONE = 0x0
36
+ TYPE_NODE = 0x1
37
+ TYPE_EDGE = 0x2
38
+ TYPE_CHUNK = 0x3
39
+
40
+ # ID PERFORMANCE BOUNDARY -- matches the engine's DENSE_ID_LIMIT (== u32::MAX). A logical_id at or below
41
+ # DENSE_ID_MAX resolves via the engine's O(1) arithmetic id-locate (flat throughput + memory at any scale);
42
+ # a larger (sparse) id spills into a disk-resident extendible hash (correct, gentler per-insert cost). For
43
+ # bulk UNIQUE-key ingest assign DENSE sequential ids (see `dense_ids`); do NOT high-bit-pack a model prefix
44
+ # (model_id<<40), which forces every id onto the sparse arm (measured ~22x slower on a single-shard ramp).
45
+ DENSE_ID_MAX = (1 << 32) - 1 # 4_294_967_295
46
+
47
+ _CRC32C_POLY = 0x82F63B78 # Castagnoli, reflected -- matches the node's CRC-32C
48
+
49
+
50
+ def _make_table() -> list[int]:
51
+ table = []
52
+ for i in range(256):
53
+ c = i
54
+ for _ in range(8):
55
+ c = (_CRC32C_POLY ^ (c >> 1)) if (c & 1) else (c >> 1)
56
+ table.append(c & 0xFFFFFFFF)
57
+ return table
58
+
59
+
60
+ _TABLE = _make_table()
61
+
62
+
63
+ def crc32c(data: bytes) -> int:
64
+ """CRC-32C (Castagnoli) over ``data`` -- the canonical Semurg container checksum."""
65
+ crc = 0xFFFFFFFF
66
+ for b in data:
67
+ crc = _TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8)
68
+ return crc ^ 0xFFFFFFFF
69
+
70
+
71
+ @dataclass
72
+ class Frame:
73
+ """A decoded frame."""
74
+
75
+ type: int
76
+ flags: int
77
+ flags_ext: int
78
+ generation: int
79
+ schema: int
80
+ logical_id: int
81
+ edge_dst_id: int
82
+ body: bytes
83
+ crc_stored: int
84
+ crc_computed: int
85
+
86
+ @property
87
+ def crc_valid(self) -> bool:
88
+ return self.crc_stored == self.crc_computed
89
+
90
+
91
+ def encode(
92
+ type: int,
93
+ *,
94
+ flags: int = 0,
95
+ flags_ext: int = 0,
96
+ generation: int = 0,
97
+ schema: int = 0,
98
+ logical_id: int = 0,
99
+ edge_dst_id: int = 0,
100
+ body: bytes = b"",
101
+ ) -> bytes:
102
+ """Encode a frame with a correct CRC-32C in bytes 60..63."""
103
+ if not 0 <= type <= 0xF:
104
+ raise ValueError("type must be a 4-bit nibble 0..15")
105
+ if not 0 <= flags <= 0xF:
106
+ raise ValueError("flags must be a 4-bit nibble 0..15")
107
+ if not 0 <= flags_ext <= 0xFF:
108
+ raise ValueError("flags_ext must be a byte 0..255")
109
+ body = bytes(body)
110
+ if len(body) > BODY_MAX:
111
+ raise ValueError(f"body is {len(body)} bytes; max {BODY_MAX} (offsets 24..59)")
112
+
113
+ head = bytearray(60)
114
+ head[0] = ((type & 0xF) << 4) | (flags & 0xF)
115
+ head[1] = flags_ext & 0xFF
116
+ head[2:6] = (generation & 0xFFFFFFFF).to_bytes(4, "little")
117
+ head[6:8] = (schema & 0xFFFF).to_bytes(2, "little")
118
+ head[8:16] = (logical_id & 0xFFFFFFFFFFFFFFFF).to_bytes(8, "little")
119
+ head[16:24] = (edge_dst_id & 0xFFFFFFFFFFFFFFFF).to_bytes(8, "little")
120
+ head[BODY_OFFSET:BODY_OFFSET + len(body)] = body
121
+ crc = crc32c(bytes(head))
122
+ return bytes(head) + crc.to_bytes(4, "little")
123
+
124
+
125
+ def decode(frame: bytes) -> Frame:
126
+ """Decode a frame into a :class:`Frame` (with ``crc_valid``)."""
127
+ if len(frame) != CONTAINER_BYTES:
128
+ raise ValueError(f"a frame is exactly {CONTAINER_BYTES} bytes, got {len(frame)}")
129
+ b0 = frame[0]
130
+ return Frame(
131
+ type=b0 >> 4,
132
+ flags=b0 & 0x0F,
133
+ flags_ext=frame[1],
134
+ generation=int.from_bytes(frame[2:6], "little"),
135
+ schema=int.from_bytes(frame[6:8], "little"),
136
+ logical_id=int.from_bytes(frame[8:16], "little"),
137
+ edge_dst_id=int.from_bytes(frame[16:24], "little"),
138
+ body=frame[BODY_OFFSET:60],
139
+ crc_stored=int.from_bytes(frame[60:64], "little"),
140
+ crc_computed=crc32c(frame[:60]),
141
+ )
142
+
143
+
144
+ def dense_ids(count: int, start: int = 0) -> range:
145
+ """A block of ``count`` DENSE sequential logical ids from ``start`` (the O(1) fast id-locate path).
146
+
147
+ Assign these to your frames' ``logical_id`` for FLAT-throughput bulk unique-key ingest. Raises if the
148
+ block would cross ``DENSE_ID_MAX`` (which would spill onto the slower disk-resident sparse hash).
149
+ """
150
+ if start < 0 or count < 0:
151
+ raise ValueError("start and count must be non-negative")
152
+ if count and start + count - 1 > DENSE_ID_MAX:
153
+ raise ValueError(
154
+ f"dense id block [{start}, {start + count}) exceeds DENSE_ID_MAX ({DENSE_ID_MAX}); "
155
+ "split across shards/stores to keep every id on the O(1) arm"
156
+ )
157
+ return range(start, start + count)
158
+
159
+
160
+ def dense_id(model: int, index: int, model_bits: int = 8) -> int:
161
+ """Pack a (model, index) pair into ONE dense logical id (<= DENSE_ID_MAX) for multi-model stores.
162
+
163
+ Interleaves ``model_bits`` low bits of ``model`` under the index, so many models share a store with
164
+ unique ids WITHOUT the ``model_id<<40`` sparse blow-up. Default 8 model bits => up to 256 models and
165
+ ~16.7M indices/model stay dense. Raises if the packed id would exceed DENSE_ID_MAX.
166
+ """
167
+ if model < 0 or index < 0 or model >= (1 << model_bits):
168
+ raise ValueError(f"model must be 0..{(1 << model_bits) - 1} and index non-negative")
169
+ packed = (index << model_bits) | model
170
+ if packed > DENSE_ID_MAX:
171
+ raise ValueError(f"packed id {packed} exceeds DENSE_ID_MAX ({DENSE_ID_MAX}); reduce indices or model_bits")
172
+ return packed
173
+
174
+
175
+ def pack(frames: list[bytes]) -> bytes:
176
+ """Concatenate encoded frames into the packed batch the /v1 endpoints accept."""
177
+ for f in frames:
178
+ if len(f) != CONTAINER_BYTES:
179
+ raise ValueError(f"every frame must be exactly {CONTAINER_BYTES} bytes")
180
+ return b"".join(frames)
181
+
182
+
183
+ def unpack(blob: bytes) -> list[bytes]:
184
+ """Split a packed blob (for example a binary fetch reply) into frames."""
185
+ if len(blob) % CONTAINER_BYTES != 0:
186
+ raise ValueError(f"blob length must be a whole multiple of {CONTAINER_BYTES} bytes")
187
+ return [blob[i:i + CONTAINER_BYTES] for i in range(0, len(blob), CONTAINER_BYTES)]
semurg/transports.py ADDED
@@ -0,0 +1,404 @@
1
+ """Pluggable transports for the Semurg client.
2
+
3
+ Three wires, one interface:
4
+
5
+ * :class:`HttpTransport` -- the working default. Every SDK method rides it today, over ONE
6
+ persistent connection per caller (HTTP/1.1 keep-alive): a call is one round trip, not a TCP
7
+ (and TLS) handshake each.
8
+ * :class:`WsTransport` -- WebSocket. The node runs a Phoenix socket for its LiveView UI;
9
+ a programmatic data-stream topic is ROADMAP. This transport builds the correct socket URL
10
+ and Phoenix join frames (real, offline-testable) and raises :class:`SemurgRoadmapError`
11
+ for request/response ops, pointing the caller at HTTP. It never fakes a live data wire.
12
+ * :class:`AtpTransport` -- Semurg ATP (Overdrive-64 frames). The frame codec is live
13
+ (see :mod:`semurg.frame`); ATP-over-TCP is ROADMAP. Unary ops raise :class:`SemurgRoadmapError`.
14
+
15
+ Only the HTTP transport is required to use the node's HTTP + /v1 API. The other two exist so a
16
+ caller can write transport-agnostic code today and flip the wire when it ships.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import base64
22
+ import http.client
23
+ import json
24
+ import select
25
+ import socket
26
+ import ssl
27
+ import threading
28
+ import time
29
+ import urllib.request
30
+ from typing import Any, Optional
31
+ from urllib.parse import unquote, urlsplit, urlunsplit
32
+
33
+ from . import frame as _frame
34
+ from .errors import (
35
+ SemurgAuthError,
36
+ SemurgConnectionError,
37
+ SemurgHTTPError,
38
+ SemurgNotFound,
39
+ SemurgRateLimited,
40
+ SemurgRoadmapError,
41
+ SemurgServerError,
42
+ SemurgTimeout,
43
+ )
44
+
45
+
46
+ def _retry_after(headers: Any, body: Any) -> Optional[float]:
47
+ ra = None
48
+ if headers is not None:
49
+ raw = headers.get("Retry-After") or headers.get("retry-after")
50
+ if raw:
51
+ try:
52
+ ra = float(raw)
53
+ except ValueError:
54
+ ra = None
55
+ if ra is None and isinstance(body, dict) and "retry_after" in body:
56
+ try:
57
+ ra = float(body["retry_after"])
58
+ except (TypeError, ValueError):
59
+ ra = None
60
+ return ra
61
+
62
+
63
+ def _raise_for_status(status: int, headers: Any, body: Any) -> None:
64
+ msg = _err_message(body, default=f"request failed ({status})")
65
+ if status in (401, 403):
66
+ raise SemurgAuthError(status, msg, body)
67
+ if status == 404:
68
+ raise SemurgNotFound(status, msg, body)
69
+ if status == 429:
70
+ raise SemurgRateLimited(status, msg, _retry_after(headers, body), body)
71
+ if status == 503:
72
+ # A warming dataset or an unavailable store: retryable, may carry Retry-After.
73
+ raise SemurgRateLimited(status, msg, _retry_after(headers, body), body)
74
+ if 500 <= status < 600:
75
+ raise SemurgServerError(status, msg, body)
76
+ if 400 <= status < 500:
77
+ raise SemurgHTTPError(status, msg, body)
78
+
79
+
80
+ def _err_message(body: Any, default: str) -> str:
81
+ if isinstance(body, dict):
82
+ err = body.get("error")
83
+ if isinstance(err, str):
84
+ detail = body.get("detail") or body.get("reason")
85
+ return f"{err}: {detail}" if detail else err
86
+ if err is not None:
87
+ return json.dumps(err)
88
+ return default
89
+
90
+
91
+ class Transport:
92
+ """Common transport interface. Subclasses implement :meth:`request`."""
93
+
94
+ name = "base"
95
+
96
+ def request(
97
+ self,
98
+ method: str,
99
+ path: str,
100
+ *,
101
+ json_body: Any = None,
102
+ raw_body: Optional[bytes] = None,
103
+ accept: str = "json",
104
+ ) -> Any:
105
+ raise NotImplementedError
106
+
107
+ def close(self) -> None:
108
+ pass
109
+
110
+
111
+ # A socket idle this long is closed, never reused: it stays below the node's own keep-alive (Bandit closes an idle
112
+ # connection after 60 s), so the client does not send a request into a socket the node is about to drop.
113
+ _IDLE_MAX_S = 30.0
114
+ # Idle connections kept per client: one per caller thread that is active at once, most recently used first.
115
+ _POOL_MAX = 8
116
+ # A REUSED idle socket the node had closed: a reset or EOF before any byte of the answer (the node dropping an idle
117
+ # connection as the request was sent). Retried once on a fresh connection; a fresh connection is never retried.
118
+ _STALE = (ConnectionResetError, ConnectionAbortedError, BrokenPipeError) # http.client.RemoteDisconnected is a ConnectionResetError
119
+
120
+
121
+ def _dropped(conn: http.client.HTTPConnection) -> bool:
122
+ """True when an idle pooled socket is no longer usable: closed, or readable while idle (EOF, a reset, a TLS
123
+ close_notify or stray bytes: nothing is owed on an idle HTTP/1.1 connection). poll, not select, so a process with
124
+ more than 1024 descriptors is judged too."""
125
+ s = conn.sock
126
+ if s is None:
127
+ return True
128
+ try:
129
+ if hasattr(select, "poll"):
130
+ p = select.poll()
131
+ p.register(s, select.POLLIN | select.POLLPRI)
132
+ return bool(p.poll(0))
133
+ return bool(select.select([s], [], [], 0)[0])
134
+ except (OSError, ValueError):
135
+ return True
136
+
137
+
138
+ class HttpTransport(Transport):
139
+ """HTTP/1.1 over the standard library (no third-party dependency), ONE PERSISTENT CONNECTION PER CALLER.
140
+
141
+ Connections are kept alive between calls in a small pool (most recently used first, at most 8 idle, a socket
142
+ idle for 30 s is closed and never reused), so a call costs one round trip instead of a TCP (+ TLS) handshake.
143
+ Thread-safe: every call in flight holds its own connection. A REUSED connection the node had already closed
144
+ (it answers nothing: a reset or EOF before the first byte of the reply) is retried ONCE on a fresh connection;
145
+ a fresh connection is never retried. Every Semurg write is idempotent by content (the same content is the same id,
146
+ reported "already"), so that one resend cannot store anything twice.
147
+
148
+ Proxies from the environment (``HTTPS_PROXY`` / ``HTTP_PROXY`` / ``NO_PROXY``) are honoured as urllib does,
149
+ through a CONNECT tunnel for an https node. Redirects are NOT followed: a 3xx raises :class:`SemurgHTTPError`
150
+ naming the new address, so the API token is never resent to a host the caller did not name.
151
+ ``connections_opened`` counts every connection this client opened (the keep-alive witness).
152
+ """
153
+
154
+ name = "http"
155
+
156
+ def __init__(self, base_url: str, token: Optional[str] = None, timeout: float = 30.0) -> None:
157
+ self.base_url = base_url.rstrip("/")
158
+ self.token = token
159
+ self.timeout = timeout
160
+ parts = urlsplit(self.base_url)
161
+ if parts.scheme not in ("http", "https") or not parts.hostname:
162
+ raise ValueError(f"base_url must be an http:// or https:// URL, got {base_url!r}")
163
+ self._https = parts.scheme == "https"
164
+ self._host = parts.hostname
165
+ self._port = parts.port or (443 if self._https else 80)
166
+ self._netloc = parts.netloc.rpartition("@")[2]
167
+ self._prefix = parts.path # a node mounted under a path (behind a reverse proxy) keeps its prefix
168
+ self._proxy = self._env_proxy()
169
+ self._ssl = ssl.create_default_context() if self._https else None
170
+ self._idle: list[tuple[http.client.HTTPConnection, float]] = []
171
+ self._lock = threading.Lock()
172
+ self.connections_opened = 0
173
+
174
+ def _env_proxy(self) -> Optional[tuple[str, int, Optional[str]]]:
175
+ """(host, port, Proxy-Authorization) of the environment's proxy for this node, by urllib's own rules."""
176
+ url = urllib.request.getproxies().get("https" if self._https else "http")
177
+ if not url or urllib.request.proxy_bypass(self._host):
178
+ return None
179
+ p = urlsplit(url if "://" in url else "http://" + url)
180
+ if not p.hostname:
181
+ return None
182
+ auth = None
183
+ if p.username is not None:
184
+ cred = f"{unquote(p.username)}:{unquote(p.password or '')}".encode("utf-8")
185
+ auth = "Basic " + base64.b64encode(cred).decode("ascii")
186
+ return (p.hostname, p.port or (443 if p.scheme == "https" else 80), auth)
187
+
188
+ def _connect(self) -> http.client.HTTPConnection:
189
+ px = self._proxy
190
+ if self._https:
191
+ if px:
192
+ conn: http.client.HTTPConnection = http.client.HTTPSConnection(px[0], px[1], timeout=self.timeout, context=self._ssl)
193
+ conn.set_tunnel(self._host, self._port, headers={"Proxy-Authorization": px[2]} if px[2] else None)
194
+ else:
195
+ conn = http.client.HTTPSConnection(self._host, self._port, timeout=self.timeout, context=self._ssl)
196
+ else:
197
+ conn = http.client.HTTPConnection(px[0] if px else self._host, px[1] if px else self._port, timeout=self.timeout)
198
+ conn.connect()
199
+ with self._lock:
200
+ self.connections_opened += 1
201
+ return conn
202
+
203
+ def _take(self) -> Optional[http.client.HTTPConnection]:
204
+ now = time.monotonic()
205
+ stale = []
206
+ conn = None
207
+ with self._lock:
208
+ while self._idle:
209
+ c, used = self._idle.pop()
210
+ if now - used <= _IDLE_MAX_S and not _dropped(c):
211
+ conn = c
212
+ break
213
+ stale.append(c)
214
+ for c in stale:
215
+ c.close()
216
+ return conn
217
+
218
+ def _put(self, conn: http.client.HTTPConnection) -> None:
219
+ with self._lock:
220
+ if len(self._idle) < _POOL_MAX:
221
+ self._idle.append((conn, time.monotonic()))
222
+ return
223
+ conn.close()
224
+
225
+ def close(self) -> None:
226
+ """Close every idle connection. The client stays usable: a later call opens a new one."""
227
+ with self._lock:
228
+ idle, self._idle = self._idle, []
229
+ for c, _ in idle:
230
+ c.close()
231
+
232
+ def _headers(self, accept: str, content_type: Optional[str]) -> dict[str, str]:
233
+ h = {"Accept": "application/octet-stream" if accept == "bin" else "application/json"}
234
+ if content_type:
235
+ h["Content-Type"] = content_type
236
+ if self.token:
237
+ h["Authorization"] = f"Bearer {self.token}"
238
+ h["User-Agent"] = "semurg-python/0.1.0"
239
+ return h
240
+
241
+ def request(
242
+ self,
243
+ method: str,
244
+ path: str,
245
+ *,
246
+ json_body: Any = None,
247
+ raw_body: Optional[bytes] = None,
248
+ accept: str = "json",
249
+ ) -> Any:
250
+ url = self.base_url + path
251
+ data: Optional[bytes] = None
252
+ content_type: Optional[str] = None
253
+ if raw_body is not None:
254
+ data = raw_body
255
+ content_type = "application/octet-stream"
256
+ elif json_body is not None:
257
+ data = json.dumps(json_body).encode("utf-8")
258
+ content_type = "application/json"
259
+
260
+ headers = self._headers(accept, content_type)
261
+ target = self._prefix + path
262
+ if self._proxy and not self._https:
263
+ target = f"http://{self._netloc}{target}" # absolute form through a plain-http proxy
264
+ if self._proxy[2]:
265
+ headers["Proxy-Authorization"] = self._proxy[2]
266
+
267
+ for attempt in (0, 1):
268
+ conn = self._take() if attempt == 0 else None
269
+ reused = conn is not None
270
+ try:
271
+ if conn is None:
272
+ conn = self._connect()
273
+ conn.request(method, target, body=data, headers=headers)
274
+ resp = conn.getresponse()
275
+ payload = resp.read()
276
+ except _STALE as e:
277
+ if conn is not None:
278
+ conn.close()
279
+ if reused:
280
+ self.close() # one idle socket the node dropped means the others went with it (a node restart)
281
+ continue
282
+ raise SemurgConnectionError(f"could not reach {url}: {e}") from e
283
+ except socket.timeout as e: # TimeoutError on 3.10+
284
+ if conn is not None:
285
+ conn.close()
286
+ raise SemurgTimeout(f"request to {url} timed out after {self.timeout}s") from e
287
+ except (OSError, http.client.HTTPException) as e:
288
+ if conn is not None:
289
+ conn.close()
290
+ raise SemurgConnectionError(f"could not reach {url}: {e}") from e
291
+ break
292
+
293
+ if resp.will_close:
294
+ conn.close()
295
+ else:
296
+ self._put(conn)
297
+ status = resp.status
298
+ if 200 <= status < 300:
299
+ return self._parse(status, resp.headers, payload, accept)
300
+ body = self._decode_error_body(resp.headers, payload)
301
+ if 300 <= status < 400:
302
+ where = resp.headers.get("Location") or "an address it did not name"
303
+ raise SemurgHTTPError(status, f"the node redirected to {where}: use that address as base_url (redirects are not followed, so the token never goes to another host)", body)
304
+ _raise_for_status(status, resp.headers, body)
305
+ raise SemurgHTTPError(status, _err_message(body, "request failed"), body)
306
+
307
+ @staticmethod
308
+ def _parse(status: int, headers: Any, payload: bytes, accept: str) -> Any:
309
+ ctype = (headers.get("Content-Type") or "").lower()
310
+ if accept == "bin" or "octet-stream" in ctype:
311
+ return payload
312
+ if not payload:
313
+ return None
314
+ try:
315
+ return json.loads(payload.decode("utf-8"))
316
+ except (ValueError, UnicodeDecodeError):
317
+ return payload
318
+
319
+ @staticmethod
320
+ def _decode_error_body(headers: Any, payload: bytes) -> Any:
321
+ ctype = (headers.get("Content-Type") or "").lower() if headers else ""
322
+ if "json" in ctype:
323
+ try:
324
+ return json.loads(payload.decode("utf-8"))
325
+ except (ValueError, UnicodeDecodeError):
326
+ pass
327
+ try:
328
+ return payload.decode("utf-8")
329
+ except UnicodeDecodeError:
330
+ return payload
331
+
332
+
333
+ class WsTransport(Transport):
334
+ """WebSocket transport. Live data-stream wire is ROADMAP; URL + Phoenix helpers are live."""
335
+
336
+ name = "ws"
337
+
338
+ def __init__(self, base_url: str, token: Optional[str] = None, timeout: float = 30.0) -> None:
339
+ self.base_url = base_url.rstrip("/")
340
+ self.token = token
341
+ self.timeout = timeout
342
+
343
+ def socket_url(self, path: str = "/live/websocket", vsn: str = "2.0.0") -> str:
344
+ """The Phoenix socket URL for this node (ws:// or wss:// to match the base scheme)."""
345
+ parts = urlsplit(self.base_url)
346
+ scheme = "wss" if parts.scheme == "https" else "ws"
347
+ query = f"vsn={vsn}"
348
+ if self.token:
349
+ query += f"&token={self.token}"
350
+ return urlunsplit((scheme, parts.netloc, path, query, ""))
351
+
352
+ @staticmethod
353
+ def join_message(topic: str, payload: Optional[dict] = None, ref: str = "1", join_ref: str = "1") -> str:
354
+ """A Phoenix v2 join frame: [join_ref, ref, topic, "phx_join", payload] as JSON."""
355
+ return json.dumps([join_ref, ref, topic, "phx_join", payload or {}])
356
+
357
+ def request(self, *args: Any, **kwargs: Any) -> Any:
358
+ raise SemurgRoadmapError(
359
+ "the programmatic WebSocket data wire is ROADMAP. The node's Phoenix socket "
360
+ f"({self.socket_url()}) currently serves the LiveView UI only. Use transport='http' "
361
+ "for data operations; use socket_url()/join_message() to wire a browser UI today."
362
+ )
363
+
364
+
365
+ class AtpTransport(Transport):
366
+ """Semurg ATP transport. Frame codec is live; ATP-over-TCP wire is ROADMAP."""
367
+
368
+ name = "atp"
369
+
370
+ # Real, offline: the frame codec (also exported as semurg.frame).
371
+ encode_frame = staticmethod(_frame.encode)
372
+ decode_frame = staticmethod(_frame.decode)
373
+ pack = staticmethod(_frame.pack)
374
+ unpack = staticmethod(_frame.unpack)
375
+ crc32c = staticmethod(_frame.crc32c)
376
+
377
+ def __init__(self, base_url: str, token: Optional[str] = None, timeout: float = 30.0, port: int = 6464) -> None:
378
+ self.base_url = base_url.rstrip("/")
379
+ self.token = token
380
+ self.timeout = timeout
381
+ self.port = port
382
+
383
+ def endpoint(self) -> str:
384
+ """The planned ATP-over-TCP endpoint (host from base_url, ATP port). ROADMAP wire."""
385
+ host = urlsplit(self.base_url).hostname or "127.0.0.1"
386
+ return f"atp://{host}:{self.port}"
387
+
388
+ def request(self, *args: Any, **kwargs: Any) -> Any:
389
+ raise SemurgRoadmapError(
390
+ "ATP-over-TCP is ROADMAP: the Overdrive-64 frame codec is live "
391
+ "(encode_frame/decode_frame/pack/unpack, unit-tested offline) but the external wire "
392
+ f"at {self.endpoint()} is not shipped. Use transport='http' for data operations."
393
+ )
394
+
395
+
396
+ def build_transport(kind: str, base_url: str, token: Optional[str], timeout: float) -> Transport:
397
+ kind = (kind or "http").lower()
398
+ if kind == "http":
399
+ return HttpTransport(base_url, token, timeout)
400
+ if kind == "ws":
401
+ return WsTransport(base_url, token, timeout)
402
+ if kind == "atp":
403
+ return AtpTransport(base_url, token, timeout)
404
+ raise ValueError(f"unknown transport {kind!r}; use 'http', 'ws' or 'atp'")
@@ -0,0 +1,83 @@
1
+ Metadata-Version: 2.4
2
+ Name: semurg
3
+ Version: 0.1.0
4
+ Summary: Python client for Semurg: memory for AI agents (knowledge graph + vector search) on your own hardware
5
+ Author: Semurg
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://one.semurg.io
8
+ Project-URL: Documentation, https://one.semurg.io/develop
9
+ Project-URL: Agent frameworks, https://one.semurg.io/agents
10
+ Keywords: semurg,ai-memory,agent-memory,knowledge-graph,vector-search,graph-rag,mcp,llm
11
+ Classifier: License :: OSI Approved :: Apache Software License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Topic :: Database
15
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
+ Requires-Python: >=3.9
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Dynamic: license-file
20
+
21
+ # semurg
22
+
23
+ The Python client for [Semurg](https://one.semurg.io): memory for AI agents (a knowledge graph and vector search in one engine) that
24
+ runs on your own hardware. Zero dependencies: it uses only the standard library.
25
+
26
+ ## 1. Start a node
27
+
28
+ With Docker on an Intel or AMD x86_64 machine (Linux, or Windows with Docker Desktop or WSL2):
29
+
30
+ ```sh
31
+ docker run -d --name semurg -p 127.0.0.1:4000:4000 \
32
+ --ulimit memlock=-1 --ulimit nofile=1048576:1048576 --security-opt seccomp=unconfined \
33
+ -v semurg-data:/var/lib/semurg one.semurg.io/semurg
34
+ until docker logs semurg 2>&1 | grep -q 'Your API token'; do sleep 1; done
35
+ export SEMURG_API_TOKEN=$(docker exec semurg cat /var/lib/semurg/.api_token)
36
+ ```
37
+
38
+ Or install a node on Ubuntu 24.04: `curl -fsSL https://one.semurg.io/install | sudo bash` ([what it needs](https://one.semurg.io/install)).
39
+ The first node is free.
40
+
41
+ ## 2. Install
42
+
43
+ ```sh
44
+ pip install semurg
45
+ ```
46
+
47
+ ## 3. Remember and recall
48
+
49
+ ```python
50
+ import os
51
+ from semurg import Client
52
+
53
+ with Client("http://127.0.0.1:4000", token=os.environ["SEMURG_API_TOKEN"]) as mem:
54
+ mem.remember(scope="demo", content="Ada Lovelace works at Acme on the analytics team.")
55
+ print(mem.recall(scope="demo", subject="Ada Lovelace")) # what is stored about one entity, with its source
56
+ print(mem.context(scope="demo", query="Where does Ada work?")) # passages and facts for a question
57
+ ```
58
+
59
+ A `scope` is a separate, named memory. Send `user_id` to keep each of your end users' memories apart.
60
+
61
+ Your agent can also send the entities and relations it found, so they are stored as a graph at once:
62
+
63
+ ```python
64
+ mem.remember(
65
+ scope="demo",
66
+ facts={"entities": [{"name": "Acme", "type": "Company"}, {"name": "Dana", "type": "Person"}],
67
+ "relations": [{"source": "Dana", "target": "Acme", "type": "WORKS_AT"}]},
68
+ passages=[{"text": "Dana works at Acme.", "mentions": ["Dana", "Acme"]}],
69
+ )
70
+ ```
71
+
72
+ ## The memory calls
73
+
74
+ `memory_status`, `remember`, `recall`, `context`, `relate`, `forget` and `delete_user`, over HTTP. The same seven tools are served
75
+ over MCP for agent frameworks: see [one.semurg.io/agents](https://one.semurg.io/agents).
76
+
77
+ ## Links
78
+
79
+ - Developer guide: https://one.semurg.io/develop
80
+ - Agent frameworks and MCP: https://one.semurg.io/agents
81
+ - Benchmarks: https://one.semurg.io/benchmark
82
+
83
+ Licence: Apache-2.0 (this client). The Semurg engine is licensed separately; the first node is free.
@@ -0,0 +1,10 @@
1
+ semurg/__init__.py,sha256=cREdA1fgZC-hr175mOs7L5HGKGk4j1WH2dyR4dyLXkI,998
2
+ semurg/client.py,sha256=T74p_OXhzfNSRYesV0Y5bTBRhBfd2Tl2HyCAPfESf-0,16900
3
+ semurg/errors.py,sha256=XviWi-hcZwCLdki3-EURpPKqOpRnvf8C6JGBefDZaYk,2342
4
+ semurg/frame.py,sha256=90XLs6mE1X2wHJ-LHcuOxv4K-nTviIcVlbWU7b1RSMM,7377
5
+ semurg/transports.py,sha256=OSSOlAyrxETATS2gKvffCVVDPOz2MegkeGOW3kJGpgk,16823
6
+ semurg-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
7
+ semurg-0.1.0.dist-info/METADATA,sha256=mGpPLQHWHhkhNvvbl8bwYYjpVQQb035cXKjkj3GLoa4,3224
8
+ semurg-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
9
+ semurg-0.1.0.dist-info/top_level.txt,sha256=LRCUgk-clKV46CK6bBWXs5Ag89mDdWFaFCqE5M5Rx7E,7
10
+ semurg-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
@@ -0,0 +1 @@
1
+ semurg