search-as-code 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,87 @@
1
+ """Milvus-lite adapter — in-process (local file), no server.
2
+
3
+ Uses the embedded Milvus-lite engine via ``MilvusClient(uri=<file>)``. Dense is
4
+ native (COSINE); keyword/regex/hybrid are served by a composed MemoryStore. String
5
+ doc ids are stored as the VARCHAR primary key.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import tempfile
12
+ from typing import Any, Optional, Sequence
13
+
14
+ from ..errors import MissingDependencyError
15
+ from ..types import Capabilities, Document, Hit, ResultSet
16
+ from .base import VectorStore
17
+ from .memory import MemoryStore
18
+
19
+
20
+ class MilvusStore(VectorStore):
21
+ backend = "milvus"
22
+
23
+ def __init__(self, dim: int, collection: str = "sac", uri: Optional[str] = None, **_: Any):
24
+ try:
25
+ from pymilvus import DataType, MilvusClient
26
+ except ImportError as e: # pragma: no cover - optional dep
27
+ raise MissingDependencyError("pymilvus", extra="search-as-code[milvus]") from e
28
+ self.dim = int(dim)
29
+ self.collection = collection
30
+ # milvus-lite treats the uri as a data dir it will create -> must NOT pre-exist
31
+ self._uri = uri or os.path.join(tempfile.mkdtemp(prefix="sac_milvus_"), "milvus.db")
32
+ self._client = MilvusClient(uri=self._uri)
33
+ self._mem = MemoryStore()
34
+ if self._client.has_collection(collection):
35
+ self._client.drop_collection(collection)
36
+ schema = self._client.create_schema(auto_id=False, enable_dynamic_field=True)
37
+ schema.add_field("id", DataType.VARCHAR, is_primary=True, max_length=512)
38
+ schema.add_field("vector", DataType.FLOAT_VECTOR, dim=self.dim)
39
+ idx = self._client.prepare_index_params()
40
+ idx.add_index(field_name="vector", metric_type="COSINE", index_type="AUTOINDEX")
41
+ self._client.create_collection(collection, schema=schema, index_params=idx)
42
+
43
+ def capabilities(self) -> Capabilities:
44
+ return Capabilities(dense=True, keyword=True, hybrid=True, regex=True,
45
+ server_side_embedding=False, native_rerank=False,
46
+ metadata_filter=True)
47
+
48
+ def upsert(self, docs: Sequence[Document]) -> None:
49
+ rows = [{"id": str(d.id), "vector": list(d.vector)} for d in docs if d.vector is not None]
50
+ B = 2000
51
+ for i in range(0, len(rows), B):
52
+ self._client.insert(self.collection, data=rows[i:i + B])
53
+ self._mem.upsert(docs)
54
+
55
+ def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
56
+ from ..filters import matches
57
+ res = self._client.search(self.collection, data=[list(vector)], anns_field="vector",
58
+ limit=top_k * (8 if flt else 1), output_fields=["id"])
59
+ hits = []
60
+ for row in res[0]:
61
+ did = row["id"] if isinstance(row, dict) else row.id
62
+ score = row["distance"] if isinstance(row, dict) else row.distance
63
+ doc = self._mem._docs.get(str(did))
64
+ if doc is None or (flt and not matches(doc.metadata, flt)):
65
+ continue
66
+ hits.append(Hit(id=str(did), score=float(score), document=doc, store=self.backend))
67
+ if len(hits) >= top_k:
68
+ break
69
+ return ResultSet(hits)
70
+
71
+ def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
72
+ return self._mem.query_keyword(text, top_k=top_k, flt=flt)
73
+
74
+ def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
75
+ return self._mem.query_regex(pattern, top_k=top_k, flt=flt)
76
+
77
+ def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
78
+ dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
79
+ kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
80
+ from ..primitives import fuse
81
+ return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
82
+
83
+ def get(self, ids: Sequence[str]) -> list[Document]:
84
+ return self._mem.get(ids)
85
+
86
+ def count(self) -> int:
87
+ return self._mem.count()
@@ -0,0 +1,102 @@
1
+ """nmslib adapter — in-process HNSW ANN, no server.
2
+
3
+ nmslib builds its graph in one shot (not incremental), so vectors are accumulated
4
+ on upsert and the index is built lazily on first query. Dense is native HNSW;
5
+ keyword/regex/hybrid are served by a composed MemoryStore. Integer point ids map
6
+ to the original doc ids (like the FAISS adapter).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Any, Sequence
12
+
13
+ import numpy as np
14
+
15
+ from ..types import Capabilities, Document, Hit, ResultSet
16
+ from .base import VectorStore
17
+ from .memory import MemoryStore
18
+
19
+
20
+ def _norm(mat: np.ndarray) -> np.ndarray:
21
+ n = np.linalg.norm(mat, axis=1, keepdims=True)
22
+ n[n == 0] = 1.0
23
+ return mat / n
24
+
25
+
26
+ class NmslibStore(VectorStore):
27
+ backend = "nmslib"
28
+
29
+ def __init__(self, dim: int, M: int = 32, ef_construction: int = 200,
30
+ ef_search: int = 200, **_: Any):
31
+ import nmslib
32
+
33
+ self._nmslib = nmslib
34
+ self.dim = int(dim)
35
+ self._ids: list[str] = []
36
+ self._vecs: list[list[float]] = []
37
+ self._mem = MemoryStore()
38
+ self._index: Any = None
39
+ self._built = False
40
+ self.params = {"M": M, "efConstruction": ef_construction}
41
+ self.ef_search = ef_search
42
+
43
+ def capabilities(self) -> Capabilities:
44
+ return Capabilities(dense=True, keyword=True, hybrid=True, regex=True,
45
+ server_side_embedding=False, native_rerank=False,
46
+ metadata_filter=True)
47
+
48
+ def upsert(self, docs: Sequence[Document]) -> None:
49
+ for d in docs:
50
+ if d.vector is None:
51
+ continue
52
+ self._ids.append(d.id)
53
+ self._vecs.append(d.vector)
54
+ self._mem.upsert(docs)
55
+ self._built = False
56
+
57
+ def _build(self) -> None:
58
+ idx = self._nmslib.init(method="hnsw", space="cosinesimil")
59
+ data = _norm(np.asarray(self._vecs, dtype=np.float32))
60
+ idx.addDataPointBatch(data, list(range(len(self._ids))))
61
+ idx.createIndex(self.params, print_progress=False)
62
+ idx.setQueryTimeParams({"efSearch": self.ef_search})
63
+ self._index = idx
64
+ self._built = True
65
+
66
+ def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
67
+ if not self._ids:
68
+ return ResultSet()
69
+ if not self._built:
70
+ self._build()
71
+ from ..filters import matches
72
+ q = _norm(np.asarray([vector], dtype=np.float32))[0]
73
+ k = min(len(self._ids), top_k * (8 if flt else 1))
74
+ pos, dist = self._index.knnQuery(q, k=k)
75
+ hits = []
76
+ for p, dd in zip(pos, dist):
77
+ doc = self._mem._docs.get(self._ids[p])
78
+ if doc is None or (flt and not matches(doc.metadata, flt)):
79
+ continue
80
+ # cosinesimil distance = 1 - cosine -> larger-is-better score
81
+ hits.append(Hit(id=self._ids[p], score=float(1.0 - dd), document=doc, store=self.backend))
82
+ if len(hits) >= top_k:
83
+ break
84
+ return ResultSet(hits)
85
+
86
+ def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
87
+ return self._mem.query_keyword(text, top_k=top_k, flt=flt)
88
+
89
+ def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
90
+ return self._mem.query_regex(pattern, top_k=top_k, flt=flt)
91
+
92
+ def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
93
+ dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
94
+ kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
95
+ from ..primitives import fuse
96
+ return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
97
+
98
+ def get(self, ids: Sequence[str]) -> list[Document]:
99
+ return self._mem.get(ids)
100
+
101
+ def count(self) -> int:
102
+ return len(self._ids)
@@ -0,0 +1,370 @@
1
+ """OpenSearch adapter. ``pip install 'search-as-code[opensearch]'`` (opensearch-py).
2
+
3
+ Implements the full retrieval surface OpenSearch does natively:
4
+ * dense kNN (HNSW ``knn_vector``) → ``query_vector``
5
+ * BM25 full-text (``match``) → ``query_keyword``
6
+ * hybrid → dense + BM25 fused with RRF (portable; no server pipeline needed)
7
+ * ``regexp`` exact/pattern search → ``query_regex``
8
+ * metadata filtering via the portable filter dialect → OpenSearch ``bool`` filter
9
+ * aggregations (``aggregate``) for the analysis-class primitives
10
+
11
+ Scores are returned larger-is-better (OpenSearch already does this for both
12
+ _score and kNN similarity), so no conversion is needed.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any, Optional, Sequence
18
+
19
+ from .._resilience import DEFAULT_BATCH_SIZE, chunked, with_retry
20
+ from ..errors import BackendError, InvalidArgumentError, MissingDependencyError
21
+ from ..filters import normalize
22
+ from ..types import Capabilities, Document, Hit, ResultSet
23
+ from .base import VectorStore
24
+
25
+ _RANGE_OPS = {"$gt": "gt", "$gte": "gte", "$lt": "lt", "$lte": "lte"}
26
+
27
+
28
+ class OpenSearchStore(VectorStore):
29
+ backend = "opensearch"
30
+
31
+ def __init__(
32
+ self,
33
+ index: str,
34
+ dim: Optional[int] = None,
35
+ hosts: Optional[list] = None,
36
+ host: str = "localhost",
37
+ port: int = 9200,
38
+ use_ssl: bool = False,
39
+ http_auth: Optional[tuple] = None,
40
+ space_type: str = "cosinesimil",
41
+ text_field: str = "text",
42
+ vector_field: str = "vector",
43
+ timeout: float = 30.0,
44
+ max_retries: int = 3,
45
+ batch_size: int = DEFAULT_BATCH_SIZE,
46
+ **client_kwargs: Any,
47
+ ):
48
+ try:
49
+ from opensearchpy import OpenSearch
50
+ except ImportError as e: # pragma: no cover - optional dep
51
+ raise MissingDependencyError("opensearch-py", extra="search-as-code[opensearch]") from e
52
+ self.index = index
53
+ self.dim = dim
54
+ self.text_field = text_field
55
+ self.vector_field = vector_field
56
+ self._space = space_type
57
+ self.batch_size = batch_size
58
+ # Lean on the client's native resilience for transient connection/timeout
59
+ # failures (retry_on_timeout), and wrap our own calls in BackendError so
60
+ # callers get one typed failure regardless of backend.
61
+ self.client = OpenSearch(
62
+ hosts=hosts or [{"host": host, "port": port}],
63
+ http_auth=http_auth,
64
+ use_ssl=use_ssl,
65
+ verify_certs=False,
66
+ ssl_show_warn=False,
67
+ timeout=timeout,
68
+ max_retries=max_retries,
69
+ retry_on_timeout=True,
70
+ **client_kwargs,
71
+ )
72
+ if dim is not None:
73
+ self.ensure_index(dim)
74
+
75
+ def _search(self, body: dict) -> dict:
76
+ """One choke point for reads: wrap transport errors as BackendError."""
77
+ try:
78
+ return self.client.search(index=self.index, body=body)
79
+ except Exception as e: # opensearchpy transport/connection errors
80
+ raise BackendError("opensearch search failed", index=self.index,
81
+ cause=type(e).__name__) from e
82
+
83
+ # ---- index lifecycle -------------------------------------------------
84
+ def ensure_index(self, dim: int) -> None:
85
+ if self.client.indices.exists(index=self.index):
86
+ return
87
+ body = {
88
+ "settings": {"index": {"knn": True, "number_of_shards": 1, "number_of_replicas": 0}},
89
+ "mappings": {
90
+ "properties": {
91
+ self.vector_field: {
92
+ "type": "knn_vector",
93
+ "dimension": dim,
94
+ "method": {
95
+ "name": "hnsw",
96
+ "engine": "lucene",
97
+ "space_type": self._space,
98
+ },
99
+ },
100
+ self.text_field: {
101
+ "type": "text",
102
+ "fields": {"keyword": {"type": "keyword", "ignore_above": 32766}},
103
+ },
104
+ }
105
+ },
106
+ }
107
+ self.client.indices.create(index=self.index, body=body)
108
+
109
+ def capabilities(self) -> Capabilities:
110
+ return Capabilities(
111
+ dense=True, keyword=True, hybrid=True, regex=True,
112
+ metadata_filter=True, native_rerank=False, server_side_embedding=False,
113
+ )
114
+
115
+ # ---- ingestion -------------------------------------------------------
116
+ def upsert(self, docs: Sequence[Document]) -> None:
117
+ from opensearchpy.helpers import bulk
118
+
119
+ actions = []
120
+ for d in docs:
121
+ src: dict[str, Any] = {**(d.metadata or {})}
122
+ if d.text is not None:
123
+ src[self.text_field] = d.text
124
+ if d.vector is not None:
125
+ src[self.vector_field] = list(d.vector)
126
+ actions.append({"_index": self.index, "_id": d.id, "_source": src})
127
+ if not actions:
128
+ return
129
+ # Batch large ingests so a single request can't blow past body-size limits,
130
+ # and retry each batch with backoff on transient failures.
131
+ batches = list(chunked(actions, self.batch_size))
132
+ for i, batch in enumerate(batches):
133
+ refresh = i == len(batches) - 1 # refresh once, on the final batch
134
+ with_retry(bulk, self.client, batch, refresh=refresh,
135
+ backend="opensearch", op="bulk")
136
+
137
+ # ---- filter translation ---------------------------------------------
138
+ def _to_filter(self, flt: Optional[dict]) -> list[dict]:
139
+ if not flt:
140
+ return []
141
+ clauses: list[dict] = []
142
+ for field, cond in normalize(flt).items():
143
+ if field.startswith("$"):
144
+ continue # nested and/or omitted in reference adapter
145
+ for op, val in cond.items():
146
+ if op == "$eq":
147
+ clauses.append({"term": {field: val}})
148
+ elif op == "$ne":
149
+ clauses.append({"bool": {"must_not": {"term": {field: val}}}})
150
+ elif op == "$in":
151
+ clauses.append({"terms": {field: list(val)}})
152
+ elif op == "$nin":
153
+ clauses.append({"bool": {"must_not": {"terms": {field: list(val)}}}})
154
+ elif op in _RANGE_OPS:
155
+ clauses.append({"range": {field: {_RANGE_OPS[op]: val}}})
156
+ return clauses
157
+
158
+ def _hits(self, resp: dict) -> ResultSet:
159
+ out = []
160
+ for h in resp["hits"]["hits"]:
161
+ src = dict(h.get("_source", {}))
162
+ text = src.pop(self.text_field, None)
163
+ src.pop(self.vector_field, None)
164
+ out.append(
165
+ Hit(
166
+ id=str(h["_id"]),
167
+ score=float(h["_score"]) if h.get("_score") is not None else 0.0,
168
+ document=Document(id=str(h["_id"]), text=text, metadata=src),
169
+ store=self.backend,
170
+ )
171
+ )
172
+ return ResultSet(out)
173
+
174
+ # ---- retrieval primitives -------------------------------------------
175
+ def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
176
+ knn = {"vector": list(vector), "k": top_k}
177
+ fclauses = self._to_filter(flt)
178
+ if fclauses:
179
+ knn["filter"] = {"bool": {"must": fclauses}}
180
+ body = {"size": top_k, "query": {"knn": {self.vector_field: knn}}}
181
+ return self._hits(self._search(body))
182
+
183
+ def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
184
+ must = [{"match": {self.text_field: text}}]
185
+ body = {"size": top_k, "query": {"bool": {"must": must, "filter": self._to_filter(flt)}}}
186
+ return self._hits(self._search(body))
187
+
188
+ def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
189
+ # Portable hybrid: run both, fuse with RRF in-SDK (no server pipeline needed).
190
+ dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
191
+ kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
192
+ from ..primitives import fuse
193
+
194
+ return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
195
+
196
+ def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
197
+ # regexp on the whole-text keyword subfield; OpenSearch anchors to the full
198
+ # value, so callers wrap with .* for substring (e.g. r".*def search.*").
199
+ body = {
200
+ "size": top_k,
201
+ "query": {"bool": {
202
+ "must": [{"regexp": {f"{self.text_field}.keyword": {"value": pattern, "flags": "ALL"}}}],
203
+ "filter": self._to_filter(flt),
204
+ }},
205
+ }
206
+ return self._hits(self._search(body))
207
+
208
+ # ---- extended lexical primitives (OpenSearch-native) -----------------
209
+ def query_phrase(self, text, top_k=10, flt=None, slop=0, field=None) -> ResultSet:
210
+ """Ordered-phrase match with optional ``slop`` (proximity). Taxonomy:
211
+ ``phrase_search`` / ``proximity_search`` / ``score_phrase_or_proximity``."""
212
+ f = field or self.text_field
213
+ body = {"size": top_k, "query": {"bool": {
214
+ "must": [{"match_phrase": {f: {"query": text, "slop": slop}}}],
215
+ "filter": self._to_filter(flt)}}}
216
+ return self._hits(self._search(body))
217
+
218
+ def query_fielded(self, text, fields, top_k=10, flt=None, type="best_fields") -> ResultSet:
219
+ """Multi-field search with per-field boosts (``fielded_search`` /
220
+ ``field_boost``). ``fields`` e.g. ``["title^2", "text"]``."""
221
+ body = {"size": top_k, "query": {"bool": {
222
+ "must": [{"multi_match": {"query": text, "fields": list(fields), "type": type}}],
223
+ "filter": self._to_filter(flt)}}}
224
+ return self._hits(self._search(body))
225
+
226
+ def query_prefix(self, prefix, top_k=10, flt=None, field=None) -> ResultSet:
227
+ """Match docs containing an indexed *term* beginning with ``prefix``
228
+ (``prefix_search``). Defaults to the analyzed text field (term-level);
229
+ pass ``field="<f>.keyword"`` for whole-value prefixing."""
230
+ f = field or self.text_field
231
+ body = {"size": top_k, "query": {"bool": {
232
+ "must": [{"prefix": {f: prefix.lower()}}], "filter": self._to_filter(flt)}}}
233
+ return self._hits(self._search(body))
234
+
235
+ def query_wildcard(self, pattern, top_k=10, flt=None, field=None) -> ResultSet:
236
+ """Glob-style wildcard match (``*``/``?``) on the keyword subfield
237
+ (``wildcard_search``)."""
238
+ f = field or f"{self.text_field}.keyword"
239
+ body = {"size": top_k, "query": {"bool": {
240
+ "must": [{"wildcard": {f: {"value": pattern}}}], "filter": self._to_filter(flt)}}}
241
+ return self._hits(self._search(body))
242
+
243
+ def query_fuzzy(self, text, top_k=10, flt=None, fuzziness="AUTO", field=None) -> ResultSet:
244
+ """Edit-distance tolerant match (``fuzzy_search``); ``fuzziness`` is an int
245
+ or ``"AUTO"``."""
246
+ f = field or self.text_field
247
+ body = {"size": top_k, "query": {"bool": {
248
+ "must": [{"match": {f: {"query": text, "fuzziness": fuzziness}}}],
249
+ "filter": self._to_filter(flt)}}}
250
+ return self._hits(self._search(body))
251
+
252
+ def more_like_this(self, text=None, ids=None, top_k=10, flt=None,
253
+ min_term_freq=1, min_doc_freq=1, fields=None) -> ResultSet:
254
+ """Find-similar / recommendation via MLT from free ``text`` and/or seed
255
+ ``ids`` already in the index (``recommendation_search``)."""
256
+ mlt: dict[str, Any] = {"fields": fields or [self.text_field],
257
+ "min_term_freq": min_term_freq, "min_doc_freq": min_doc_freq}
258
+ like: list[Any] = []
259
+ if text:
260
+ like.append(text)
261
+ if ids:
262
+ like.extend({"_index": self.index, "_id": str(i)} for i in ids)
263
+ if not like:
264
+ raise InvalidArgumentError("more_like_this needs text and/or ids")
265
+ mlt["like"] = like
266
+ body = {"size": top_k, "query": {"bool": {
267
+ "must": [{"more_like_this": mlt}], "filter": self._to_filter(flt)}}}
268
+ return self._hits(self._search(body))
269
+
270
+ def random_sample(self, size=10, flt=None, seed=None) -> ResultSet:
271
+ """Random (optionally seeded, reproducible) sample of the filtered corpus
272
+ (``random_sample``)."""
273
+ rnd: dict[str, Any] = {}
274
+ if seed is not None:
275
+ rnd = {"seed": seed, "field": "_seq_no"}
276
+ body = {"size": size, "query": {"function_score": {
277
+ "query": {"bool": {"filter": self._to_filter(flt)}},
278
+ "random_score": rnd}}}
279
+ return self._hits(self._search(body))
280
+
281
+ def browse(self, top_k=10, flt=None, sort_field=None, after=None) -> ResultSet:
282
+ """Query-less enumeration of the (filtered) corpus with cursor paging via
283
+ ``search_after`` (``browse_or_scroll``). Sorts by ``sort_field`` or ``_id``."""
284
+ sort = [{sort_field: "asc"}] if sort_field else [{"_seq_no": "asc"}]
285
+ body: dict[str, Any] = {"size": top_k, "sort": sort,
286
+ "query": {"bool": {"filter": self._to_filter(flt)}}}
287
+ if after is not None:
288
+ body["search_after"] = after
289
+ return self._hits(self._search(body))
290
+
291
+ # ---- analysis-class primitive ---------------------------------------
292
+ def aggregate(self, aggs: dict, flt: Optional[dict] = None, size: int = 0) -> dict:
293
+ """Run a native OpenSearch aggregation (facets, stats, group-by)."""
294
+ body: dict[str, Any] = {"size": size, "aggs": aggs}
295
+ fclauses = self._to_filter(flt)
296
+ if fclauses:
297
+ body["query"] = {"bool": {"filter": fclauses}}
298
+ return self._search(body).get("aggregations", {})
299
+
300
+ def facet(self, field, size=20, flt=None) -> dict[str, int]:
301
+ """Categorical value counts for ``field`` (``facet``). Returns {value: count}."""
302
+ agg = self.aggregate({"f": {"terms": {"field": field, "size": size}}}, flt=flt)
303
+ return {b["key"]: b["doc_count"] for b in agg.get("f", {}).get("buckets", [])}
304
+
305
+ def count_distinct(self, field, flt=None) -> int:
306
+ """Approximate distinct-value count for ``field`` (``count_distinct``)."""
307
+ agg = self.aggregate({"c": {"cardinality": {"field": field}}}, flt=flt)
308
+ return int(agg.get("c", {}).get("value", 0))
309
+
310
+ def stats(self, field, flt=None) -> dict[str, float]:
311
+ """min/max/avg/sum/count for a numeric ``field`` (``aggregate_statistics``)."""
312
+ agg = self.aggregate({"s": {"stats": {"field": field}}}, flt=flt)
313
+ return agg.get("s", {})
314
+
315
+ # ---- fetch / admin ---------------------------------------------------
316
+ def get(self, ids: Sequence[str]) -> list[Document]:
317
+ if not ids:
318
+ return []
319
+ resp = self.client.mget(index=self.index, body={"ids": list(ids)})
320
+ out = []
321
+ for d in resp["docs"]:
322
+ if not d.get("found"):
323
+ continue
324
+ src = dict(d.get("_source", {}))
325
+ text = src.pop(self.text_field, None)
326
+ src.pop(self.vector_field, None)
327
+ out.append(Document(id=str(d["_id"]), text=text, metadata=src))
328
+ return out
329
+
330
+ def delete(self, ids: Sequence[str]) -> None:
331
+ for i in ids:
332
+ try:
333
+ self.client.delete(index=self.index, id=i, refresh=True)
334
+ except Exception:
335
+ pass
336
+
337
+ def count(self) -> int:
338
+ return int(self.client.count(index=self.index)["count"])
339
+
340
+ def sample(self, n: int = 5) -> list[Document]:
341
+ body = {"size": n, "query": {"function_score": {"query": {"match_all": {}},
342
+ "random_score": {}}}}
343
+ out = []
344
+ for h in self.client.search(index=self.index, body=body)["hits"]["hits"]:
345
+ src = dict(h.get("_source", {}))
346
+ text = src.pop(self.text_field, None)
347
+ src.pop(self.vector_field, None)
348
+ out.append(Document(id=str(h["_id"]), text=text, metadata=src))
349
+ return out
350
+
351
+ def describe_schema(self) -> dict:
352
+ """Fields -> ES types (+ a couple sample docs) so the agent knows the shape first."""
353
+ m = self.client.indices.get_mapping(index=self.index)
354
+ props = next(iter(m.values()))["mappings"].get("properties", {})
355
+
356
+ def flatten(p, pre=""):
357
+ out = {}
358
+ for k, v in p.items():
359
+ if v.get("type"):
360
+ out[pre + k] = v["type"]
361
+ if "properties" in v:
362
+ out.update(flatten(v["properties"], pre + k + "."))
363
+ return out
364
+
365
+ fields = flatten(props)
366
+ samples = self.sample(2)
367
+ return {"index": self.index, "count": self.count(),
368
+ "text_field": self.text_field, "vector_field": self.vector_field,
369
+ "n_fields": len(fields), "fields": fields,
370
+ "sample_text": [(d.text or "")[:200] for d in samples]}
@@ -0,0 +1,110 @@
1
+ """pgvector adapter. ``pip install 'search-as-code[pgvector]'``.
2
+
3
+ Postgres + the pgvector extension. Metadata lives in a JSONB column; the
4
+ portable filter dialect is translated to parameterized SQL. Cosine *distance*
5
+ (``<=>``) is converted to a larger-is-better similarity.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from typing import Any, Optional, Sequence
12
+
13
+ from ..errors import MissingDependencyError
14
+ from ..filters import normalize
15
+ from ..types import Capabilities, Document, Hit, ResultSet
16
+ from .base import VectorStore
17
+
18
+ _SQL_OP = {"$eq": "=", "$ne": "<>", "$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<="}
19
+
20
+
21
+ class PgVectorStore(VectorStore):
22
+ backend = "pgvector"
23
+
24
+ def __init__(self, dsn: str, table: str = "sac_docs", dim: int = 256, **_: Any):
25
+ try:
26
+ import psycopg
27
+ from pgvector.psycopg import register_vector
28
+ except ImportError as e: # pragma: no cover - optional dep
29
+ raise MissingDependencyError("psycopg", extra="search-as-code[pgvector]") from e
30
+ self._conn = psycopg.connect(dsn, autocommit=True)
31
+ self._conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
32
+ register_vector(self._conn)
33
+ self.table = table
34
+ self._conn.execute(
35
+ f"CREATE TABLE IF NOT EXISTS {table} "
36
+ f"(id TEXT PRIMARY KEY, text TEXT, metadata JSONB, embedding vector({dim}))"
37
+ )
38
+
39
+ def capabilities(self) -> Capabilities:
40
+ # Postgres also has full-text search; wire query_keyword() to tsvector to
41
+ # flip `keyword`/`hybrid` on. Left dense-only for the reference adapter.
42
+ return Capabilities(dense=True, keyword=False, hybrid=False, metadata_filter=True)
43
+
44
+ def upsert(self, docs: Sequence[Document]) -> None:
45
+ rows = [
46
+ (d.id, d.text, json.dumps(d.metadata or {}), d.vector)
47
+ for d in docs
48
+ if d.vector is not None
49
+ ]
50
+ if not rows:
51
+ return
52
+ with self._conn.cursor() as cur:
53
+ cur.executemany(
54
+ f"INSERT INTO {self.table} (id, text, metadata, embedding) VALUES (%s,%s,%s,%s) "
55
+ f"ON CONFLICT (id) DO UPDATE SET text=EXCLUDED.text, "
56
+ f"metadata=EXCLUDED.metadata, embedding=EXCLUDED.embedding",
57
+ rows,
58
+ )
59
+
60
+ def _where(self, flt: Optional[dict]) -> tuple[str, list]:
61
+ if not flt:
62
+ return "", []
63
+ clauses: list[str] = []
64
+ params: list[Any] = []
65
+ for field_name, cond in normalize(flt).items():
66
+ if field_name.startswith("$"):
67
+ continue
68
+ for op, val in cond.items():
69
+ if op in _SQL_OP:
70
+ clauses.append(f"metadata->>%s {_SQL_OP[op]} %s")
71
+ params.extend([field_name, str(val)])
72
+ elif op == "$in":
73
+ clauses.append("metadata->>%s = ANY(%s)")
74
+ params.extend([field_name, [str(v) for v in val]])
75
+ return (" WHERE " + " AND ".join(clauses)) if clauses else "", params
76
+
77
+ def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
78
+ import numpy as np
79
+
80
+ where, params = self._where(flt)
81
+ sql = (
82
+ f"SELECT id, text, metadata, 1 - (embedding <=> %s) AS score "
83
+ f"FROM {self.table}{where} ORDER BY embedding <=> %s LIMIT %s"
84
+ )
85
+ vec = np.asarray(vector, dtype=np.float32)
86
+ rows = self._conn.execute(sql, [vec, *params, vec, top_k]).fetchall()
87
+ return ResultSet(
88
+ Hit(
89
+ id=r[0],
90
+ score=float(r[3]),
91
+ document=Document(id=r[0], text=r[1], metadata=r[2] or {}),
92
+ store=self.backend,
93
+ )
94
+ for r in rows
95
+ )
96
+
97
+ def get(self, ids: Sequence[str]) -> list[Document]:
98
+ rows = self._conn.execute(
99
+ f"SELECT id, text, metadata FROM {self.table} WHERE id = ANY(%s)", [list(ids)]
100
+ ).fetchall()
101
+ return [Document(id=r[0], text=r[1], metadata=r[2] or {}) for r in rows]
102
+
103
+ def delete(self, ids: Sequence[str]) -> None:
104
+ self._conn.execute(f"DELETE FROM {self.table} WHERE id = ANY(%s)", [list(ids)])
105
+
106
+ def count(self) -> int:
107
+ return self._conn.execute(f"SELECT COUNT(*) FROM {self.table}").fetchone()[0]
108
+
109
+ def close(self) -> None:
110
+ self._conn.close()