search-as-code 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- search_as_code/__init__.py +89 -0
- search_as_code/_resilience.py +99 -0
- search_as_code/adapters/__init__.py +5 -0
- search_as_code/adapters/base.py +108 -0
- search_as_code/adapters/chroma.py +95 -0
- search_as_code/adapters/faiss_store.py +90 -0
- search_as_code/adapters/memory.py +148 -0
- search_as_code/adapters/milvus_store.py +87 -0
- search_as_code/adapters/nmslib_store.py +102 -0
- search_as_code/adapters/opensearch.py +370 -0
- search_as_code/adapters/pgvector.py +110 -0
- search_as_code/adapters/qdrant.py +133 -0
- search_as_code/adapters/registry.py +98 -0
- search_as_code/adapters/sqlite_store.py +105 -0
- search_as_code/embeddings.py +155 -0
- search_as_code/errors.py +145 -0
- search_as_code/explore/__init__.py +31 -0
- search_as_code/explore/engine.py +160 -0
- search_as_code/explore/pack.py +127 -0
- search_as_code/explore/report.py +72 -0
- search_as_code/explore/stages.py +349 -0
- search_as_code/filters.py +102 -0
- search_as_code/primitives.py +422 -0
- search_as_code/rerankers.py +94 -0
- search_as_code/sandbox.py +116 -0
- search_as_code/session.py +460 -0
- search_as_code/types.py +129 -0
- search_as_code-0.0.1.dist-info/METADATA +234 -0
- search_as_code-0.0.1.dist-info/RECORD +32 -0
- search_as_code-0.0.1.dist-info/WHEEL +5 -0
- search_as_code-0.0.1.dist-info/licenses/LICENSE +201 -0
- search_as_code-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Milvus-lite adapter — in-process (local file), no server.
|
|
2
|
+
|
|
3
|
+
Uses the embedded Milvus-lite engine via ``MilvusClient(uri=<file>)``. Dense is
|
|
4
|
+
native (COSINE); keyword/regex/hybrid are served by a composed MemoryStore. String
|
|
5
|
+
doc ids are stored as the VARCHAR primary key.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import tempfile
|
|
12
|
+
from typing import Any, Optional, Sequence
|
|
13
|
+
|
|
14
|
+
from ..errors import MissingDependencyError
|
|
15
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
16
|
+
from .base import VectorStore
|
|
17
|
+
from .memory import MemoryStore
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class MilvusStore(VectorStore):
|
|
21
|
+
backend = "milvus"
|
|
22
|
+
|
|
23
|
+
def __init__(self, dim: int, collection: str = "sac", uri: Optional[str] = None, **_: Any):
|
|
24
|
+
try:
|
|
25
|
+
from pymilvus import DataType, MilvusClient
|
|
26
|
+
except ImportError as e: # pragma: no cover - optional dep
|
|
27
|
+
raise MissingDependencyError("pymilvus", extra="search-as-code[milvus]") from e
|
|
28
|
+
self.dim = int(dim)
|
|
29
|
+
self.collection = collection
|
|
30
|
+
# milvus-lite treats the uri as a data dir it will create -> must NOT pre-exist
|
|
31
|
+
self._uri = uri or os.path.join(tempfile.mkdtemp(prefix="sac_milvus_"), "milvus.db")
|
|
32
|
+
self._client = MilvusClient(uri=self._uri)
|
|
33
|
+
self._mem = MemoryStore()
|
|
34
|
+
if self._client.has_collection(collection):
|
|
35
|
+
self._client.drop_collection(collection)
|
|
36
|
+
schema = self._client.create_schema(auto_id=False, enable_dynamic_field=True)
|
|
37
|
+
schema.add_field("id", DataType.VARCHAR, is_primary=True, max_length=512)
|
|
38
|
+
schema.add_field("vector", DataType.FLOAT_VECTOR, dim=self.dim)
|
|
39
|
+
idx = self._client.prepare_index_params()
|
|
40
|
+
idx.add_index(field_name="vector", metric_type="COSINE", index_type="AUTOINDEX")
|
|
41
|
+
self._client.create_collection(collection, schema=schema, index_params=idx)
|
|
42
|
+
|
|
43
|
+
def capabilities(self) -> Capabilities:
|
|
44
|
+
return Capabilities(dense=True, keyword=True, hybrid=True, regex=True,
|
|
45
|
+
server_side_embedding=False, native_rerank=False,
|
|
46
|
+
metadata_filter=True)
|
|
47
|
+
|
|
48
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
49
|
+
rows = [{"id": str(d.id), "vector": list(d.vector)} for d in docs if d.vector is not None]
|
|
50
|
+
B = 2000
|
|
51
|
+
for i in range(0, len(rows), B):
|
|
52
|
+
self._client.insert(self.collection, data=rows[i:i + B])
|
|
53
|
+
self._mem.upsert(docs)
|
|
54
|
+
|
|
55
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
56
|
+
from ..filters import matches
|
|
57
|
+
res = self._client.search(self.collection, data=[list(vector)], anns_field="vector",
|
|
58
|
+
limit=top_k * (8 if flt else 1), output_fields=["id"])
|
|
59
|
+
hits = []
|
|
60
|
+
for row in res[0]:
|
|
61
|
+
did = row["id"] if isinstance(row, dict) else row.id
|
|
62
|
+
score = row["distance"] if isinstance(row, dict) else row.distance
|
|
63
|
+
doc = self._mem._docs.get(str(did))
|
|
64
|
+
if doc is None or (flt and not matches(doc.metadata, flt)):
|
|
65
|
+
continue
|
|
66
|
+
hits.append(Hit(id=str(did), score=float(score), document=doc, store=self.backend))
|
|
67
|
+
if len(hits) >= top_k:
|
|
68
|
+
break
|
|
69
|
+
return ResultSet(hits)
|
|
70
|
+
|
|
71
|
+
def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
|
|
72
|
+
return self._mem.query_keyword(text, top_k=top_k, flt=flt)
|
|
73
|
+
|
|
74
|
+
def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
|
|
75
|
+
return self._mem.query_regex(pattern, top_k=top_k, flt=flt)
|
|
76
|
+
|
|
77
|
+
def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
|
|
78
|
+
dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
|
|
79
|
+
kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
|
|
80
|
+
from ..primitives import fuse
|
|
81
|
+
return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
|
|
82
|
+
|
|
83
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
84
|
+
return self._mem.get(ids)
|
|
85
|
+
|
|
86
|
+
def count(self) -> int:
|
|
87
|
+
return self._mem.count()
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""nmslib adapter — in-process HNSW ANN, no server.
|
|
2
|
+
|
|
3
|
+
nmslib builds its graph in one shot (not incremental), so vectors are accumulated
|
|
4
|
+
on upsert and the index is built lazily on first query. Dense is native HNSW;
|
|
5
|
+
keyword/regex/hybrid are served by a composed MemoryStore. Integer point ids map
|
|
6
|
+
to the original doc ids (like the FAISS adapter).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any, Sequence
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
16
|
+
from .base import VectorStore
|
|
17
|
+
from .memory import MemoryStore
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _norm(mat: np.ndarray) -> np.ndarray:
|
|
21
|
+
n = np.linalg.norm(mat, axis=1, keepdims=True)
|
|
22
|
+
n[n == 0] = 1.0
|
|
23
|
+
return mat / n
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class NmslibStore(VectorStore):
|
|
27
|
+
backend = "nmslib"
|
|
28
|
+
|
|
29
|
+
def __init__(self, dim: int, M: int = 32, ef_construction: int = 200,
|
|
30
|
+
ef_search: int = 200, **_: Any):
|
|
31
|
+
import nmslib
|
|
32
|
+
|
|
33
|
+
self._nmslib = nmslib
|
|
34
|
+
self.dim = int(dim)
|
|
35
|
+
self._ids: list[str] = []
|
|
36
|
+
self._vecs: list[list[float]] = []
|
|
37
|
+
self._mem = MemoryStore()
|
|
38
|
+
self._index: Any = None
|
|
39
|
+
self._built = False
|
|
40
|
+
self.params = {"M": M, "efConstruction": ef_construction}
|
|
41
|
+
self.ef_search = ef_search
|
|
42
|
+
|
|
43
|
+
def capabilities(self) -> Capabilities:
|
|
44
|
+
return Capabilities(dense=True, keyword=True, hybrid=True, regex=True,
|
|
45
|
+
server_side_embedding=False, native_rerank=False,
|
|
46
|
+
metadata_filter=True)
|
|
47
|
+
|
|
48
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
49
|
+
for d in docs:
|
|
50
|
+
if d.vector is None:
|
|
51
|
+
continue
|
|
52
|
+
self._ids.append(d.id)
|
|
53
|
+
self._vecs.append(d.vector)
|
|
54
|
+
self._mem.upsert(docs)
|
|
55
|
+
self._built = False
|
|
56
|
+
|
|
57
|
+
def _build(self) -> None:
|
|
58
|
+
idx = self._nmslib.init(method="hnsw", space="cosinesimil")
|
|
59
|
+
data = _norm(np.asarray(self._vecs, dtype=np.float32))
|
|
60
|
+
idx.addDataPointBatch(data, list(range(len(self._ids))))
|
|
61
|
+
idx.createIndex(self.params, print_progress=False)
|
|
62
|
+
idx.setQueryTimeParams({"efSearch": self.ef_search})
|
|
63
|
+
self._index = idx
|
|
64
|
+
self._built = True
|
|
65
|
+
|
|
66
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
67
|
+
if not self._ids:
|
|
68
|
+
return ResultSet()
|
|
69
|
+
if not self._built:
|
|
70
|
+
self._build()
|
|
71
|
+
from ..filters import matches
|
|
72
|
+
q = _norm(np.asarray([vector], dtype=np.float32))[0]
|
|
73
|
+
k = min(len(self._ids), top_k * (8 if flt else 1))
|
|
74
|
+
pos, dist = self._index.knnQuery(q, k=k)
|
|
75
|
+
hits = []
|
|
76
|
+
for p, dd in zip(pos, dist):
|
|
77
|
+
doc = self._mem._docs.get(self._ids[p])
|
|
78
|
+
if doc is None or (flt and not matches(doc.metadata, flt)):
|
|
79
|
+
continue
|
|
80
|
+
# cosinesimil distance = 1 - cosine -> larger-is-better score
|
|
81
|
+
hits.append(Hit(id=self._ids[p], score=float(1.0 - dd), document=doc, store=self.backend))
|
|
82
|
+
if len(hits) >= top_k:
|
|
83
|
+
break
|
|
84
|
+
return ResultSet(hits)
|
|
85
|
+
|
|
86
|
+
def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
|
|
87
|
+
return self._mem.query_keyword(text, top_k=top_k, flt=flt)
|
|
88
|
+
|
|
89
|
+
def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
|
|
90
|
+
return self._mem.query_regex(pattern, top_k=top_k, flt=flt)
|
|
91
|
+
|
|
92
|
+
def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
|
|
93
|
+
dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
|
|
94
|
+
kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
|
|
95
|
+
from ..primitives import fuse
|
|
96
|
+
return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
|
|
97
|
+
|
|
98
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
99
|
+
return self._mem.get(ids)
|
|
100
|
+
|
|
101
|
+
def count(self) -> int:
|
|
102
|
+
return len(self._ids)
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""OpenSearch adapter. ``pip install 'search-as-code[opensearch]'`` (opensearch-py).
|
|
2
|
+
|
|
3
|
+
Implements the full retrieval surface OpenSearch does natively:
|
|
4
|
+
* dense kNN (HNSW ``knn_vector``) → ``query_vector``
|
|
5
|
+
* BM25 full-text (``match``) → ``query_keyword``
|
|
6
|
+
* hybrid → dense + BM25 fused with RRF (portable; no server pipeline needed)
|
|
7
|
+
* ``regexp`` exact/pattern search → ``query_regex``
|
|
8
|
+
* metadata filtering via the portable filter dialect → OpenSearch ``bool`` filter
|
|
9
|
+
* aggregations (``aggregate``) for the analysis-class primitives
|
|
10
|
+
|
|
11
|
+
Scores are returned larger-is-better (OpenSearch already does this for both
|
|
12
|
+
_score and kNN similarity), so no conversion is needed.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any, Optional, Sequence
|
|
18
|
+
|
|
19
|
+
from .._resilience import DEFAULT_BATCH_SIZE, chunked, with_retry
|
|
20
|
+
from ..errors import BackendError, InvalidArgumentError, MissingDependencyError
|
|
21
|
+
from ..filters import normalize
|
|
22
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
23
|
+
from .base import VectorStore
|
|
24
|
+
|
|
25
|
+
_RANGE_OPS = {"$gt": "gt", "$gte": "gte", "$lt": "lt", "$lte": "lte"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class OpenSearchStore(VectorStore):
|
|
29
|
+
backend = "opensearch"
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
index: str,
|
|
34
|
+
dim: Optional[int] = None,
|
|
35
|
+
hosts: Optional[list] = None,
|
|
36
|
+
host: str = "localhost",
|
|
37
|
+
port: int = 9200,
|
|
38
|
+
use_ssl: bool = False,
|
|
39
|
+
http_auth: Optional[tuple] = None,
|
|
40
|
+
space_type: str = "cosinesimil",
|
|
41
|
+
text_field: str = "text",
|
|
42
|
+
vector_field: str = "vector",
|
|
43
|
+
timeout: float = 30.0,
|
|
44
|
+
max_retries: int = 3,
|
|
45
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
46
|
+
**client_kwargs: Any,
|
|
47
|
+
):
|
|
48
|
+
try:
|
|
49
|
+
from opensearchpy import OpenSearch
|
|
50
|
+
except ImportError as e: # pragma: no cover - optional dep
|
|
51
|
+
raise MissingDependencyError("opensearch-py", extra="search-as-code[opensearch]") from e
|
|
52
|
+
self.index = index
|
|
53
|
+
self.dim = dim
|
|
54
|
+
self.text_field = text_field
|
|
55
|
+
self.vector_field = vector_field
|
|
56
|
+
self._space = space_type
|
|
57
|
+
self.batch_size = batch_size
|
|
58
|
+
# Lean on the client's native resilience for transient connection/timeout
|
|
59
|
+
# failures (retry_on_timeout), and wrap our own calls in BackendError so
|
|
60
|
+
# callers get one typed failure regardless of backend.
|
|
61
|
+
self.client = OpenSearch(
|
|
62
|
+
hosts=hosts or [{"host": host, "port": port}],
|
|
63
|
+
http_auth=http_auth,
|
|
64
|
+
use_ssl=use_ssl,
|
|
65
|
+
verify_certs=False,
|
|
66
|
+
ssl_show_warn=False,
|
|
67
|
+
timeout=timeout,
|
|
68
|
+
max_retries=max_retries,
|
|
69
|
+
retry_on_timeout=True,
|
|
70
|
+
**client_kwargs,
|
|
71
|
+
)
|
|
72
|
+
if dim is not None:
|
|
73
|
+
self.ensure_index(dim)
|
|
74
|
+
|
|
75
|
+
def _search(self, body: dict) -> dict:
|
|
76
|
+
"""One choke point for reads: wrap transport errors as BackendError."""
|
|
77
|
+
try:
|
|
78
|
+
return self.client.search(index=self.index, body=body)
|
|
79
|
+
except Exception as e: # opensearchpy transport/connection errors
|
|
80
|
+
raise BackendError("opensearch search failed", index=self.index,
|
|
81
|
+
cause=type(e).__name__) from e
|
|
82
|
+
|
|
83
|
+
# ---- index lifecycle -------------------------------------------------
|
|
84
|
+
def ensure_index(self, dim: int) -> None:
|
|
85
|
+
if self.client.indices.exists(index=self.index):
|
|
86
|
+
return
|
|
87
|
+
body = {
|
|
88
|
+
"settings": {"index": {"knn": True, "number_of_shards": 1, "number_of_replicas": 0}},
|
|
89
|
+
"mappings": {
|
|
90
|
+
"properties": {
|
|
91
|
+
self.vector_field: {
|
|
92
|
+
"type": "knn_vector",
|
|
93
|
+
"dimension": dim,
|
|
94
|
+
"method": {
|
|
95
|
+
"name": "hnsw",
|
|
96
|
+
"engine": "lucene",
|
|
97
|
+
"space_type": self._space,
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
self.text_field: {
|
|
101
|
+
"type": "text",
|
|
102
|
+
"fields": {"keyword": {"type": "keyword", "ignore_above": 32766}},
|
|
103
|
+
},
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
self.client.indices.create(index=self.index, body=body)
|
|
108
|
+
|
|
109
|
+
def capabilities(self) -> Capabilities:
|
|
110
|
+
return Capabilities(
|
|
111
|
+
dense=True, keyword=True, hybrid=True, regex=True,
|
|
112
|
+
metadata_filter=True, native_rerank=False, server_side_embedding=False,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# ---- ingestion -------------------------------------------------------
|
|
116
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
117
|
+
from opensearchpy.helpers import bulk
|
|
118
|
+
|
|
119
|
+
actions = []
|
|
120
|
+
for d in docs:
|
|
121
|
+
src: dict[str, Any] = {**(d.metadata or {})}
|
|
122
|
+
if d.text is not None:
|
|
123
|
+
src[self.text_field] = d.text
|
|
124
|
+
if d.vector is not None:
|
|
125
|
+
src[self.vector_field] = list(d.vector)
|
|
126
|
+
actions.append({"_index": self.index, "_id": d.id, "_source": src})
|
|
127
|
+
if not actions:
|
|
128
|
+
return
|
|
129
|
+
# Batch large ingests so a single request can't blow past body-size limits,
|
|
130
|
+
# and retry each batch with backoff on transient failures.
|
|
131
|
+
batches = list(chunked(actions, self.batch_size))
|
|
132
|
+
for i, batch in enumerate(batches):
|
|
133
|
+
refresh = i == len(batches) - 1 # refresh once, on the final batch
|
|
134
|
+
with_retry(bulk, self.client, batch, refresh=refresh,
|
|
135
|
+
backend="opensearch", op="bulk")
|
|
136
|
+
|
|
137
|
+
# ---- filter translation ---------------------------------------------
|
|
138
|
+
def _to_filter(self, flt: Optional[dict]) -> list[dict]:
|
|
139
|
+
if not flt:
|
|
140
|
+
return []
|
|
141
|
+
clauses: list[dict] = []
|
|
142
|
+
for field, cond in normalize(flt).items():
|
|
143
|
+
if field.startswith("$"):
|
|
144
|
+
continue # nested and/or omitted in reference adapter
|
|
145
|
+
for op, val in cond.items():
|
|
146
|
+
if op == "$eq":
|
|
147
|
+
clauses.append({"term": {field: val}})
|
|
148
|
+
elif op == "$ne":
|
|
149
|
+
clauses.append({"bool": {"must_not": {"term": {field: val}}}})
|
|
150
|
+
elif op == "$in":
|
|
151
|
+
clauses.append({"terms": {field: list(val)}})
|
|
152
|
+
elif op == "$nin":
|
|
153
|
+
clauses.append({"bool": {"must_not": {"terms": {field: list(val)}}}})
|
|
154
|
+
elif op in _RANGE_OPS:
|
|
155
|
+
clauses.append({"range": {field: {_RANGE_OPS[op]: val}}})
|
|
156
|
+
return clauses
|
|
157
|
+
|
|
158
|
+
def _hits(self, resp: dict) -> ResultSet:
|
|
159
|
+
out = []
|
|
160
|
+
for h in resp["hits"]["hits"]:
|
|
161
|
+
src = dict(h.get("_source", {}))
|
|
162
|
+
text = src.pop(self.text_field, None)
|
|
163
|
+
src.pop(self.vector_field, None)
|
|
164
|
+
out.append(
|
|
165
|
+
Hit(
|
|
166
|
+
id=str(h["_id"]),
|
|
167
|
+
score=float(h["_score"]) if h.get("_score") is not None else 0.0,
|
|
168
|
+
document=Document(id=str(h["_id"]), text=text, metadata=src),
|
|
169
|
+
store=self.backend,
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
return ResultSet(out)
|
|
173
|
+
|
|
174
|
+
# ---- retrieval primitives -------------------------------------------
|
|
175
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
176
|
+
knn = {"vector": list(vector), "k": top_k}
|
|
177
|
+
fclauses = self._to_filter(flt)
|
|
178
|
+
if fclauses:
|
|
179
|
+
knn["filter"] = {"bool": {"must": fclauses}}
|
|
180
|
+
body = {"size": top_k, "query": {"knn": {self.vector_field: knn}}}
|
|
181
|
+
return self._hits(self._search(body))
|
|
182
|
+
|
|
183
|
+
def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
|
|
184
|
+
must = [{"match": {self.text_field: text}}]
|
|
185
|
+
body = {"size": top_k, "query": {"bool": {"must": must, "filter": self._to_filter(flt)}}}
|
|
186
|
+
return self._hits(self._search(body))
|
|
187
|
+
|
|
188
|
+
def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
|
|
189
|
+
# Portable hybrid: run both, fuse with RRF in-SDK (no server pipeline needed).
|
|
190
|
+
dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
|
|
191
|
+
kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
|
|
192
|
+
from ..primitives import fuse
|
|
193
|
+
|
|
194
|
+
return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
|
|
195
|
+
|
|
196
|
+
def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
|
|
197
|
+
# regexp on the whole-text keyword subfield; OpenSearch anchors to the full
|
|
198
|
+
# value, so callers wrap with .* for substring (e.g. r".*def search.*").
|
|
199
|
+
body = {
|
|
200
|
+
"size": top_k,
|
|
201
|
+
"query": {"bool": {
|
|
202
|
+
"must": [{"regexp": {f"{self.text_field}.keyword": {"value": pattern, "flags": "ALL"}}}],
|
|
203
|
+
"filter": self._to_filter(flt),
|
|
204
|
+
}},
|
|
205
|
+
}
|
|
206
|
+
return self._hits(self._search(body))
|
|
207
|
+
|
|
208
|
+
# ---- extended lexical primitives (OpenSearch-native) -----------------
|
|
209
|
+
def query_phrase(self, text, top_k=10, flt=None, slop=0, field=None) -> ResultSet:
|
|
210
|
+
"""Ordered-phrase match with optional ``slop`` (proximity). Taxonomy:
|
|
211
|
+
``phrase_search`` / ``proximity_search`` / ``score_phrase_or_proximity``."""
|
|
212
|
+
f = field or self.text_field
|
|
213
|
+
body = {"size": top_k, "query": {"bool": {
|
|
214
|
+
"must": [{"match_phrase": {f: {"query": text, "slop": slop}}}],
|
|
215
|
+
"filter": self._to_filter(flt)}}}
|
|
216
|
+
return self._hits(self._search(body))
|
|
217
|
+
|
|
218
|
+
def query_fielded(self, text, fields, top_k=10, flt=None, type="best_fields") -> ResultSet:
|
|
219
|
+
"""Multi-field search with per-field boosts (``fielded_search`` /
|
|
220
|
+
``field_boost``). ``fields`` e.g. ``["title^2", "text"]``."""
|
|
221
|
+
body = {"size": top_k, "query": {"bool": {
|
|
222
|
+
"must": [{"multi_match": {"query": text, "fields": list(fields), "type": type}}],
|
|
223
|
+
"filter": self._to_filter(flt)}}}
|
|
224
|
+
return self._hits(self._search(body))
|
|
225
|
+
|
|
226
|
+
def query_prefix(self, prefix, top_k=10, flt=None, field=None) -> ResultSet:
|
|
227
|
+
"""Match docs containing an indexed *term* beginning with ``prefix``
|
|
228
|
+
(``prefix_search``). Defaults to the analyzed text field (term-level);
|
|
229
|
+
pass ``field="<f>.keyword"`` for whole-value prefixing."""
|
|
230
|
+
f = field or self.text_field
|
|
231
|
+
body = {"size": top_k, "query": {"bool": {
|
|
232
|
+
"must": [{"prefix": {f: prefix.lower()}}], "filter": self._to_filter(flt)}}}
|
|
233
|
+
return self._hits(self._search(body))
|
|
234
|
+
|
|
235
|
+
def query_wildcard(self, pattern, top_k=10, flt=None, field=None) -> ResultSet:
|
|
236
|
+
"""Glob-style wildcard match (``*``/``?``) on the keyword subfield
|
|
237
|
+
(``wildcard_search``)."""
|
|
238
|
+
f = field or f"{self.text_field}.keyword"
|
|
239
|
+
body = {"size": top_k, "query": {"bool": {
|
|
240
|
+
"must": [{"wildcard": {f: {"value": pattern}}}], "filter": self._to_filter(flt)}}}
|
|
241
|
+
return self._hits(self._search(body))
|
|
242
|
+
|
|
243
|
+
def query_fuzzy(self, text, top_k=10, flt=None, fuzziness="AUTO", field=None) -> ResultSet:
|
|
244
|
+
"""Edit-distance tolerant match (``fuzzy_search``); ``fuzziness`` is an int
|
|
245
|
+
or ``"AUTO"``."""
|
|
246
|
+
f = field or self.text_field
|
|
247
|
+
body = {"size": top_k, "query": {"bool": {
|
|
248
|
+
"must": [{"match": {f: {"query": text, "fuzziness": fuzziness}}}],
|
|
249
|
+
"filter": self._to_filter(flt)}}}
|
|
250
|
+
return self._hits(self._search(body))
|
|
251
|
+
|
|
252
|
+
def more_like_this(self, text=None, ids=None, top_k=10, flt=None,
|
|
253
|
+
min_term_freq=1, min_doc_freq=1, fields=None) -> ResultSet:
|
|
254
|
+
"""Find-similar / recommendation via MLT from free ``text`` and/or seed
|
|
255
|
+
``ids`` already in the index (``recommendation_search``)."""
|
|
256
|
+
mlt: dict[str, Any] = {"fields": fields or [self.text_field],
|
|
257
|
+
"min_term_freq": min_term_freq, "min_doc_freq": min_doc_freq}
|
|
258
|
+
like: list[Any] = []
|
|
259
|
+
if text:
|
|
260
|
+
like.append(text)
|
|
261
|
+
if ids:
|
|
262
|
+
like.extend({"_index": self.index, "_id": str(i)} for i in ids)
|
|
263
|
+
if not like:
|
|
264
|
+
raise InvalidArgumentError("more_like_this needs text and/or ids")
|
|
265
|
+
mlt["like"] = like
|
|
266
|
+
body = {"size": top_k, "query": {"bool": {
|
|
267
|
+
"must": [{"more_like_this": mlt}], "filter": self._to_filter(flt)}}}
|
|
268
|
+
return self._hits(self._search(body))
|
|
269
|
+
|
|
270
|
+
def random_sample(self, size=10, flt=None, seed=None) -> ResultSet:
|
|
271
|
+
"""Random (optionally seeded, reproducible) sample of the filtered corpus
|
|
272
|
+
(``random_sample``)."""
|
|
273
|
+
rnd: dict[str, Any] = {}
|
|
274
|
+
if seed is not None:
|
|
275
|
+
rnd = {"seed": seed, "field": "_seq_no"}
|
|
276
|
+
body = {"size": size, "query": {"function_score": {
|
|
277
|
+
"query": {"bool": {"filter": self._to_filter(flt)}},
|
|
278
|
+
"random_score": rnd}}}
|
|
279
|
+
return self._hits(self._search(body))
|
|
280
|
+
|
|
281
|
+
def browse(self, top_k=10, flt=None, sort_field=None, after=None) -> ResultSet:
|
|
282
|
+
"""Query-less enumeration of the (filtered) corpus with cursor paging via
|
|
283
|
+
``search_after`` (``browse_or_scroll``). Sorts by ``sort_field`` or ``_id``."""
|
|
284
|
+
sort = [{sort_field: "asc"}] if sort_field else [{"_seq_no": "asc"}]
|
|
285
|
+
body: dict[str, Any] = {"size": top_k, "sort": sort,
|
|
286
|
+
"query": {"bool": {"filter": self._to_filter(flt)}}}
|
|
287
|
+
if after is not None:
|
|
288
|
+
body["search_after"] = after
|
|
289
|
+
return self._hits(self._search(body))
|
|
290
|
+
|
|
291
|
+
# ---- analysis-class primitive ---------------------------------------
|
|
292
|
+
def aggregate(self, aggs: dict, flt: Optional[dict] = None, size: int = 0) -> dict:
|
|
293
|
+
"""Run a native OpenSearch aggregation (facets, stats, group-by)."""
|
|
294
|
+
body: dict[str, Any] = {"size": size, "aggs": aggs}
|
|
295
|
+
fclauses = self._to_filter(flt)
|
|
296
|
+
if fclauses:
|
|
297
|
+
body["query"] = {"bool": {"filter": fclauses}}
|
|
298
|
+
return self._search(body).get("aggregations", {})
|
|
299
|
+
|
|
300
|
+
def facet(self, field, size=20, flt=None) -> dict[str, int]:
|
|
301
|
+
"""Categorical value counts for ``field`` (``facet``). Returns {value: count}."""
|
|
302
|
+
agg = self.aggregate({"f": {"terms": {"field": field, "size": size}}}, flt=flt)
|
|
303
|
+
return {b["key"]: b["doc_count"] for b in agg.get("f", {}).get("buckets", [])}
|
|
304
|
+
|
|
305
|
+
def count_distinct(self, field, flt=None) -> int:
|
|
306
|
+
"""Approximate distinct-value count for ``field`` (``count_distinct``)."""
|
|
307
|
+
agg = self.aggregate({"c": {"cardinality": {"field": field}}}, flt=flt)
|
|
308
|
+
return int(agg.get("c", {}).get("value", 0))
|
|
309
|
+
|
|
310
|
+
def stats(self, field, flt=None) -> dict[str, float]:
|
|
311
|
+
"""min/max/avg/sum/count for a numeric ``field`` (``aggregate_statistics``)."""
|
|
312
|
+
agg = self.aggregate({"s": {"stats": {"field": field}}}, flt=flt)
|
|
313
|
+
return agg.get("s", {})
|
|
314
|
+
|
|
315
|
+
# ---- fetch / admin ---------------------------------------------------
|
|
316
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
317
|
+
if not ids:
|
|
318
|
+
return []
|
|
319
|
+
resp = self.client.mget(index=self.index, body={"ids": list(ids)})
|
|
320
|
+
out = []
|
|
321
|
+
for d in resp["docs"]:
|
|
322
|
+
if not d.get("found"):
|
|
323
|
+
continue
|
|
324
|
+
src = dict(d.get("_source", {}))
|
|
325
|
+
text = src.pop(self.text_field, None)
|
|
326
|
+
src.pop(self.vector_field, None)
|
|
327
|
+
out.append(Document(id=str(d["_id"]), text=text, metadata=src))
|
|
328
|
+
return out
|
|
329
|
+
|
|
330
|
+
def delete(self, ids: Sequence[str]) -> None:
|
|
331
|
+
for i in ids:
|
|
332
|
+
try:
|
|
333
|
+
self.client.delete(index=self.index, id=i, refresh=True)
|
|
334
|
+
except Exception:
|
|
335
|
+
pass
|
|
336
|
+
|
|
337
|
+
def count(self) -> int:
|
|
338
|
+
return int(self.client.count(index=self.index)["count"])
|
|
339
|
+
|
|
340
|
+
def sample(self, n: int = 5) -> list[Document]:
|
|
341
|
+
body = {"size": n, "query": {"function_score": {"query": {"match_all": {}},
|
|
342
|
+
"random_score": {}}}}
|
|
343
|
+
out = []
|
|
344
|
+
for h in self.client.search(index=self.index, body=body)["hits"]["hits"]:
|
|
345
|
+
src = dict(h.get("_source", {}))
|
|
346
|
+
text = src.pop(self.text_field, None)
|
|
347
|
+
src.pop(self.vector_field, None)
|
|
348
|
+
out.append(Document(id=str(h["_id"]), text=text, metadata=src))
|
|
349
|
+
return out
|
|
350
|
+
|
|
351
|
+
def describe_schema(self) -> dict:
|
|
352
|
+
"""Fields -> ES types (+ a couple sample docs) so the agent knows the shape first."""
|
|
353
|
+
m = self.client.indices.get_mapping(index=self.index)
|
|
354
|
+
props = next(iter(m.values()))["mappings"].get("properties", {})
|
|
355
|
+
|
|
356
|
+
def flatten(p, pre=""):
|
|
357
|
+
out = {}
|
|
358
|
+
for k, v in p.items():
|
|
359
|
+
if v.get("type"):
|
|
360
|
+
out[pre + k] = v["type"]
|
|
361
|
+
if "properties" in v:
|
|
362
|
+
out.update(flatten(v["properties"], pre + k + "."))
|
|
363
|
+
return out
|
|
364
|
+
|
|
365
|
+
fields = flatten(props)
|
|
366
|
+
samples = self.sample(2)
|
|
367
|
+
return {"index": self.index, "count": self.count(),
|
|
368
|
+
"text_field": self.text_field, "vector_field": self.vector_field,
|
|
369
|
+
"n_fields": len(fields), "fields": fields,
|
|
370
|
+
"sample_text": [(d.text or "")[:200] for d in samples]}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""pgvector adapter. ``pip install 'search-as-code[pgvector]'``.
|
|
2
|
+
|
|
3
|
+
Postgres + the pgvector extension. Metadata lives in a JSONB column; the
|
|
4
|
+
portable filter dialect is translated to parameterized SQL. Cosine *distance*
|
|
5
|
+
(``<=>``) is converted to a larger-is-better similarity.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from typing import Any, Optional, Sequence
|
|
12
|
+
|
|
13
|
+
from ..errors import MissingDependencyError
|
|
14
|
+
from ..filters import normalize
|
|
15
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
16
|
+
from .base import VectorStore
|
|
17
|
+
|
|
18
|
+
_SQL_OP = {"$eq": "=", "$ne": "<>", "$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<="}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PgVectorStore(VectorStore):
|
|
22
|
+
backend = "pgvector"
|
|
23
|
+
|
|
24
|
+
def __init__(self, dsn: str, table: str = "sac_docs", dim: int = 256, **_: Any):
|
|
25
|
+
try:
|
|
26
|
+
import psycopg
|
|
27
|
+
from pgvector.psycopg import register_vector
|
|
28
|
+
except ImportError as e: # pragma: no cover - optional dep
|
|
29
|
+
raise MissingDependencyError("psycopg", extra="search-as-code[pgvector]") from e
|
|
30
|
+
self._conn = psycopg.connect(dsn, autocommit=True)
|
|
31
|
+
self._conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
32
|
+
register_vector(self._conn)
|
|
33
|
+
self.table = table
|
|
34
|
+
self._conn.execute(
|
|
35
|
+
f"CREATE TABLE IF NOT EXISTS {table} "
|
|
36
|
+
f"(id TEXT PRIMARY KEY, text TEXT, metadata JSONB, embedding vector({dim}))"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
def capabilities(self) -> Capabilities:
|
|
40
|
+
# Postgres also has full-text search; wire query_keyword() to tsvector to
|
|
41
|
+
# flip `keyword`/`hybrid` on. Left dense-only for the reference adapter.
|
|
42
|
+
return Capabilities(dense=True, keyword=False, hybrid=False, metadata_filter=True)
|
|
43
|
+
|
|
44
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
45
|
+
rows = [
|
|
46
|
+
(d.id, d.text, json.dumps(d.metadata or {}), d.vector)
|
|
47
|
+
for d in docs
|
|
48
|
+
if d.vector is not None
|
|
49
|
+
]
|
|
50
|
+
if not rows:
|
|
51
|
+
return
|
|
52
|
+
with self._conn.cursor() as cur:
|
|
53
|
+
cur.executemany(
|
|
54
|
+
f"INSERT INTO {self.table} (id, text, metadata, embedding) VALUES (%s,%s,%s,%s) "
|
|
55
|
+
f"ON CONFLICT (id) DO UPDATE SET text=EXCLUDED.text, "
|
|
56
|
+
f"metadata=EXCLUDED.metadata, embedding=EXCLUDED.embedding",
|
|
57
|
+
rows,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def _where(self, flt: Optional[dict]) -> tuple[str, list]:
|
|
61
|
+
if not flt:
|
|
62
|
+
return "", []
|
|
63
|
+
clauses: list[str] = []
|
|
64
|
+
params: list[Any] = []
|
|
65
|
+
for field_name, cond in normalize(flt).items():
|
|
66
|
+
if field_name.startswith("$"):
|
|
67
|
+
continue
|
|
68
|
+
for op, val in cond.items():
|
|
69
|
+
if op in _SQL_OP:
|
|
70
|
+
clauses.append(f"metadata->>%s {_SQL_OP[op]} %s")
|
|
71
|
+
params.extend([field_name, str(val)])
|
|
72
|
+
elif op == "$in":
|
|
73
|
+
clauses.append("metadata->>%s = ANY(%s)")
|
|
74
|
+
params.extend([field_name, [str(v) for v in val]])
|
|
75
|
+
return (" WHERE " + " AND ".join(clauses)) if clauses else "", params
|
|
76
|
+
|
|
77
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
78
|
+
import numpy as np
|
|
79
|
+
|
|
80
|
+
where, params = self._where(flt)
|
|
81
|
+
sql = (
|
|
82
|
+
f"SELECT id, text, metadata, 1 - (embedding <=> %s) AS score "
|
|
83
|
+
f"FROM {self.table}{where} ORDER BY embedding <=> %s LIMIT %s"
|
|
84
|
+
)
|
|
85
|
+
vec = np.asarray(vector, dtype=np.float32)
|
|
86
|
+
rows = self._conn.execute(sql, [vec, *params, vec, top_k]).fetchall()
|
|
87
|
+
return ResultSet(
|
|
88
|
+
Hit(
|
|
89
|
+
id=r[0],
|
|
90
|
+
score=float(r[3]),
|
|
91
|
+
document=Document(id=r[0], text=r[1], metadata=r[2] or {}),
|
|
92
|
+
store=self.backend,
|
|
93
|
+
)
|
|
94
|
+
for r in rows
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
98
|
+
rows = self._conn.execute(
|
|
99
|
+
f"SELECT id, text, metadata FROM {self.table} WHERE id = ANY(%s)", [list(ids)]
|
|
100
|
+
).fetchall()
|
|
101
|
+
return [Document(id=r[0], text=r[1], metadata=r[2] or {}) for r in rows]
|
|
102
|
+
|
|
103
|
+
def delete(self, ids: Sequence[str]) -> None:
|
|
104
|
+
self._conn.execute(f"DELETE FROM {self.table} WHERE id = ANY(%s)", [list(ids)])
|
|
105
|
+
|
|
106
|
+
def count(self) -> int:
|
|
107
|
+
return self._conn.execute(f"SELECT COUNT(*) FROM {self.table}").fetchone()[0]
|
|
108
|
+
|
|
109
|
+
def close(self) -> None:
|
|
110
|
+
self._conn.close()
|