search-as-code 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- search_as_code/__init__.py +89 -0
- search_as_code/_resilience.py +99 -0
- search_as_code/adapters/__init__.py +5 -0
- search_as_code/adapters/base.py +108 -0
- search_as_code/adapters/chroma.py +95 -0
- search_as_code/adapters/faiss_store.py +90 -0
- search_as_code/adapters/memory.py +148 -0
- search_as_code/adapters/milvus_store.py +87 -0
- search_as_code/adapters/nmslib_store.py +102 -0
- search_as_code/adapters/opensearch.py +370 -0
- search_as_code/adapters/pgvector.py +110 -0
- search_as_code/adapters/qdrant.py +133 -0
- search_as_code/adapters/registry.py +98 -0
- search_as_code/adapters/sqlite_store.py +105 -0
- search_as_code/embeddings.py +155 -0
- search_as_code/errors.py +145 -0
- search_as_code/explore/__init__.py +31 -0
- search_as_code/explore/engine.py +160 -0
- search_as_code/explore/pack.py +127 -0
- search_as_code/explore/report.py +72 -0
- search_as_code/explore/stages.py +349 -0
- search_as_code/filters.py +102 -0
- search_as_code/primitives.py +422 -0
- search_as_code/rerankers.py +94 -0
- search_as_code/sandbox.py +116 -0
- search_as_code/session.py +460 -0
- search_as_code/types.py +129 -0
- search_as_code-0.0.1.dist-info/METADATA +234 -0
- search_as_code-0.0.1.dist-info/RECORD +32 -0
- search_as_code-0.0.1.dist-info/WHEEL +5 -0
- search_as_code-0.0.1.dist-info/licenses/LICENSE +201 -0
- search_as_code-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Qdrant adapter. ``pip install 'search-as-code[qdrant]'``.
|
|
2
|
+
|
|
3
|
+
Demonstrates the pattern every real adapter follows: translate the portable
|
|
4
|
+
filter dialect to the backend DSL, convert native distances to larger-is-better
|
|
5
|
+
scores, and declare capabilities honestly.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import uuid
|
|
11
|
+
from typing import Any, Optional, Sequence
|
|
12
|
+
|
|
13
|
+
from .._resilience import DEFAULT_BATCH_SIZE, chunked
|
|
14
|
+
from ..errors import MissingDependencyError
|
|
15
|
+
from ..filters import normalize
|
|
16
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
17
|
+
from .base import VectorStore
|
|
18
|
+
|
|
19
|
+
_OP_MAP = {"$gt": "gt", "$gte": "gte", "$lt": "lt", "$lte": "lte"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class QdrantStore(VectorStore):
|
|
23
|
+
backend = "qdrant"
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
collection: str,
|
|
28
|
+
url: Optional[str] = None,
|
|
29
|
+
location: Optional[str] = ":memory:",
|
|
30
|
+
dim: Optional[int] = None,
|
|
31
|
+
distance: str = "Cosine",
|
|
32
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
33
|
+
**client_kwargs: Any,
|
|
34
|
+
):
|
|
35
|
+
try:
|
|
36
|
+
from qdrant_client import QdrantClient
|
|
37
|
+
from qdrant_client.http import models as qm
|
|
38
|
+
except ImportError as e: # pragma: no cover - optional dep
|
|
39
|
+
raise MissingDependencyError("qdrant-client", extra="search-as-code[qdrant]") from e
|
|
40
|
+
self._qm = qm
|
|
41
|
+
self._client = QdrantClient(url=url, location=None if url else location, **client_kwargs)
|
|
42
|
+
self.collection = collection
|
|
43
|
+
self.batch_size = batch_size
|
|
44
|
+
self._dim = dim
|
|
45
|
+
self._distance = distance
|
|
46
|
+
if dim and not self._client.collection_exists(collection):
|
|
47
|
+
self._client.create_collection(
|
|
48
|
+
collection,
|
|
49
|
+
vectors_config=qm.VectorParams(size=dim, distance=getattr(qm.Distance, distance.upper())),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def capabilities(self) -> Capabilities:
|
|
53
|
+
return Capabilities(dense=True, keyword=False, hybrid=False, metadata_filter=True)
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def _pid(doc_id: str) -> Any:
|
|
57
|
+
"""Qdrant point ids must be unsigned int or UUID. Pass ints through;
|
|
58
|
+
map arbitrary strings to a deterministic uuid5 (original kept in payload)."""
|
|
59
|
+
s = str(doc_id)
|
|
60
|
+
if s.isdigit():
|
|
61
|
+
return int(s)
|
|
62
|
+
return str(uuid.uuid5(uuid.NAMESPACE_URL, s))
|
|
63
|
+
|
|
64
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
65
|
+
qm = self._qm
|
|
66
|
+
points = [
|
|
67
|
+
qm.PointStruct(id=self._pid(d.id), vector=d.vector,
|
|
68
|
+
payload={"text": d.text, "_sac_id": str(d.id), **d.metadata})
|
|
69
|
+
for d in docs
|
|
70
|
+
if d.vector is not None
|
|
71
|
+
]
|
|
72
|
+
for batch in chunked(points, self.batch_size):
|
|
73
|
+
self._client.upsert(self.collection, points=batch)
|
|
74
|
+
|
|
75
|
+
def _to_filter(self, flt: Optional[dict]) -> Any:
|
|
76
|
+
if not flt:
|
|
77
|
+
return None
|
|
78
|
+
qm = self._qm
|
|
79
|
+
must = []
|
|
80
|
+
for field_name, cond in normalize(flt).items():
|
|
81
|
+
if field_name.startswith("$"):
|
|
82
|
+
continue # nested and/or omitted for brevity in the reference adapter
|
|
83
|
+
for op, val in cond.items():
|
|
84
|
+
if op == "$eq":
|
|
85
|
+
must.append(qm.FieldCondition(key=field_name, match=qm.MatchValue(value=val)))
|
|
86
|
+
elif op == "$in":
|
|
87
|
+
must.append(qm.FieldCondition(key=field_name, match=qm.MatchAny(any=val)))
|
|
88
|
+
elif op in _OP_MAP:
|
|
89
|
+
must.append(qm.FieldCondition(key=field_name, range=qm.Range(**{_OP_MAP[op]: val})))
|
|
90
|
+
return qm.Filter(must=must) if must else None
|
|
91
|
+
|
|
92
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
93
|
+
flt_ = self._to_filter(flt)
|
|
94
|
+
if hasattr(self._client, "query_points"): # qdrant-client >= 1.10
|
|
95
|
+
res = self._client.query_points(
|
|
96
|
+
self.collection, query=list(vector), limit=top_k,
|
|
97
|
+
query_filter=flt_, with_payload=True,
|
|
98
|
+
).points
|
|
99
|
+
else: # older clients
|
|
100
|
+
res = self._client.search(
|
|
101
|
+
self.collection, query_vector=list(vector), limit=top_k,
|
|
102
|
+
query_filter=flt_, with_payload=True,
|
|
103
|
+
)
|
|
104
|
+
hits = []
|
|
105
|
+
for p in res:
|
|
106
|
+
payload = dict(p.payload or {})
|
|
107
|
+
text = payload.pop("text", None)
|
|
108
|
+
did = payload.pop("_sac_id", str(p.id)) # restore the original id
|
|
109
|
+
hits.append(
|
|
110
|
+
Hit(
|
|
111
|
+
id=did,
|
|
112
|
+
score=float(p.score), # Qdrant cosine similarity: larger is better
|
|
113
|
+
document=Document(id=did, text=text, metadata=payload),
|
|
114
|
+
store=self.backend,
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
return ResultSet(hits)
|
|
118
|
+
|
|
119
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
120
|
+
recs = self._client.retrieve(self.collection, ids=[self._pid(i) for i in ids], with_payload=True)
|
|
121
|
+
out = []
|
|
122
|
+
for r in recs:
|
|
123
|
+
payload = dict(r.payload or {})
|
|
124
|
+
text = payload.pop("text", None)
|
|
125
|
+
did = payload.pop("_sac_id", str(r.id))
|
|
126
|
+
out.append(Document(id=did, text=text, metadata=payload))
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
def delete(self, ids: Sequence[str]) -> None:
|
|
130
|
+
self._client.delete(self.collection, points_selector=[self._pid(i) for i in ids])
|
|
131
|
+
|
|
132
|
+
def count(self) -> int:
|
|
133
|
+
return self._client.count(self.collection).count
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Backend registry — ``connect(backend, **opts)`` returns a uniform store.
|
|
2
|
+
|
|
3
|
+
Real adapters are imported lazily so their heavy client libraries are only
|
|
4
|
+
required when actually used. Registering a custom backend is one call:
|
|
5
|
+
|
|
6
|
+
from search_as_code.adapters import register
|
|
7
|
+
register("mystore", MyStore)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Any, Callable
|
|
13
|
+
|
|
14
|
+
from ..errors import BackendNotFoundError
|
|
15
|
+
from .base import VectorStore
|
|
16
|
+
from .memory import MemoryStore
|
|
17
|
+
|
|
18
|
+
# Lazy factories keep optional deps out of the base install.
|
|
19
|
+
_LAZY: dict[str, Callable[..., VectorStore]] = {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _load_qdrant(**opts: Any) -> VectorStore:
|
|
23
|
+
from .qdrant import QdrantStore
|
|
24
|
+
|
|
25
|
+
return QdrantStore(**opts)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _load_chroma(**opts: Any) -> VectorStore:
|
|
29
|
+
from .chroma import ChromaStore
|
|
30
|
+
|
|
31
|
+
return ChromaStore(**opts)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load_pgvector(**opts: Any) -> VectorStore:
|
|
35
|
+
from .pgvector import PgVectorStore
|
|
36
|
+
|
|
37
|
+
return PgVectorStore(**opts)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _load_opensearch(**opts: Any) -> VectorStore:
|
|
41
|
+
from .opensearch import OpenSearchStore
|
|
42
|
+
|
|
43
|
+
return OpenSearchStore(**opts)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _load_faiss(**opts: Any) -> VectorStore:
|
|
47
|
+
from .faiss_store import FaissStore
|
|
48
|
+
|
|
49
|
+
return FaissStore(**opts)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _load_sqlite(**opts: Any) -> VectorStore:
|
|
53
|
+
from .sqlite_store import SqliteStore
|
|
54
|
+
|
|
55
|
+
return SqliteStore(**opts)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _load_nmslib(**opts: Any) -> VectorStore:
|
|
59
|
+
from .nmslib_store import NmslibStore
|
|
60
|
+
|
|
61
|
+
return NmslibStore(**opts)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _load_milvus(**opts: Any) -> VectorStore:
|
|
65
|
+
from .milvus_store import MilvusStore
|
|
66
|
+
|
|
67
|
+
return MilvusStore(**opts)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
_REGISTRY: dict[str, Callable[..., VectorStore]] = {
|
|
71
|
+
"memory": MemoryStore,
|
|
72
|
+
"qdrant": _load_qdrant,
|
|
73
|
+
"chroma": _load_chroma,
|
|
74
|
+
"pgvector": _load_pgvector,
|
|
75
|
+
"opensearch": _load_opensearch,
|
|
76
|
+
"faiss": _load_faiss,
|
|
77
|
+
"sqlite": _load_sqlite,
|
|
78
|
+
"nmslib": _load_nmslib,
|
|
79
|
+
"milvus": _load_milvus,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def register(name: str, factory: Callable[..., VectorStore]) -> None:
|
|
84
|
+
_REGISTRY[name] = factory
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def available() -> list[str]:
|
|
88
|
+
return sorted(_REGISTRY)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def connect(backend: str = "memory", **opts: Any) -> VectorStore:
|
|
92
|
+
try:
|
|
93
|
+
factory = _REGISTRY[backend]
|
|
94
|
+
except KeyError:
|
|
95
|
+
raise BackendNotFoundError(
|
|
96
|
+
"unknown backend", backend=backend, available=available()
|
|
97
|
+
) from None
|
|
98
|
+
return factory(**opts)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""SQLite adapter — the "you don't even need a vector DB" reference.
|
|
2
|
+
|
|
3
|
+
Vectors are stored as float32 BLOBs in an ordinary SQLite table; dense search is
|
|
4
|
+
brute-force cosine in numpy. Persistent, zero-server, stdlib-only. Keyword/regex
|
|
5
|
+
run over the stored text. Proves the primitive API works over a plain SQL store.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sqlite3
|
|
11
|
+
from typing import Any, Optional, Sequence
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from ..types import Capabilities, Document, Hit, ResultSet
|
|
16
|
+
from .base import VectorStore
|
|
17
|
+
from .memory import MemoryStore
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SqliteStore(VectorStore):
|
|
21
|
+
backend = "sqlite"
|
|
22
|
+
|
|
23
|
+
def __init__(self, dim: int, path: str = ":memory:", **_: Any):
|
|
24
|
+
self.dim = int(dim)
|
|
25
|
+
self.db = sqlite3.connect(path)
|
|
26
|
+
self.db.execute(
|
|
27
|
+
"CREATE TABLE IF NOT EXISTS docs "
|
|
28
|
+
"(id TEXT PRIMARY KEY, text TEXT, meta TEXT, vec BLOB)"
|
|
29
|
+
)
|
|
30
|
+
self.db.commit()
|
|
31
|
+
self._mem = MemoryStore() # keyword/regex + fast doc cache
|
|
32
|
+
self._mat: Optional[np.ndarray] = None
|
|
33
|
+
self._ids: list[str] = []
|
|
34
|
+
self._dirty = True
|
|
35
|
+
|
|
36
|
+
def capabilities(self) -> Capabilities:
|
|
37
|
+
return Capabilities(dense=True, keyword=True, hybrid=True, regex=True,
|
|
38
|
+
server_side_embedding=False, native_rerank=False,
|
|
39
|
+
metadata_filter=True)
|
|
40
|
+
|
|
41
|
+
def upsert(self, docs: Sequence[Document]) -> None:
|
|
42
|
+
import json
|
|
43
|
+
rows = []
|
|
44
|
+
for d in docs:
|
|
45
|
+
vec = None if d.vector is None else np.asarray(d.vector, dtype=np.float32).tobytes()
|
|
46
|
+
rows.append((d.id, d.text or "", json.dumps(d.metadata or {}), vec))
|
|
47
|
+
self.db.executemany("INSERT OR REPLACE INTO docs VALUES (?,?,?,?)", rows)
|
|
48
|
+
self.db.commit()
|
|
49
|
+
self._mem.upsert(docs)
|
|
50
|
+
self._dirty = True
|
|
51
|
+
|
|
52
|
+
def _rebuild(self) -> None:
|
|
53
|
+
self._ids, vecs = [], []
|
|
54
|
+
for did, blob in self.db.execute("SELECT id, vec FROM docs WHERE vec IS NOT NULL"):
|
|
55
|
+
self._ids.append(did)
|
|
56
|
+
vecs.append(np.frombuffer(blob, dtype=np.float32))
|
|
57
|
+
if vecs:
|
|
58
|
+
mat = np.vstack(vecs)
|
|
59
|
+
n = np.linalg.norm(mat, axis=1, keepdims=True)
|
|
60
|
+
n[n == 0] = 1.0
|
|
61
|
+
self._mat = mat / n
|
|
62
|
+
else:
|
|
63
|
+
self._mat = None
|
|
64
|
+
self._dirty = False
|
|
65
|
+
|
|
66
|
+
def query_vector(self, vector, top_k=10, flt=None) -> ResultSet:
|
|
67
|
+
if self._dirty:
|
|
68
|
+
self._rebuild()
|
|
69
|
+
if self._mat is None:
|
|
70
|
+
return ResultSet()
|
|
71
|
+
from ..filters import matches
|
|
72
|
+
q = np.asarray(vector, dtype=np.float32)
|
|
73
|
+
q = q / (np.linalg.norm(q) or 1.0)
|
|
74
|
+
sims = self._mat @ q
|
|
75
|
+
order = np.argsort(-sims)
|
|
76
|
+
hits = []
|
|
77
|
+
for i in order:
|
|
78
|
+
doc = self._mem._docs.get(self._ids[i])
|
|
79
|
+
if doc is None or (flt and not matches(doc.metadata, flt)):
|
|
80
|
+
continue
|
|
81
|
+
hits.append(Hit(id=self._ids[i], score=float(sims[i]), document=doc, store=self.backend))
|
|
82
|
+
if len(hits) >= top_k:
|
|
83
|
+
break
|
|
84
|
+
return ResultSet(hits)
|
|
85
|
+
|
|
86
|
+
def query_keyword(self, text, top_k=10, flt=None) -> ResultSet:
|
|
87
|
+
return self._mem.query_keyword(text, top_k=top_k, flt=flt)
|
|
88
|
+
|
|
89
|
+
def query_regex(self, pattern, top_k=10, flt=None) -> ResultSet:
|
|
90
|
+
return self._mem.query_regex(pattern, top_k=top_k, flt=flt)
|
|
91
|
+
|
|
92
|
+
def query_hybrid(self, vector, text, top_k=10, flt=None, alpha=0.5) -> ResultSet:
|
|
93
|
+
dense = self.query_vector(vector, top_k=top_k * 4, flt=flt)
|
|
94
|
+
kw = self.query_keyword(text, top_k=top_k * 4, flt=flt)
|
|
95
|
+
from ..primitives import fuse
|
|
96
|
+
return fuse([dense, kw], weights=[alpha, 1 - alpha]).top(top_k)
|
|
97
|
+
|
|
98
|
+
def get(self, ids: Sequence[str]) -> list[Document]:
|
|
99
|
+
return self._mem.get(ids)
|
|
100
|
+
|
|
101
|
+
def count(self) -> int:
|
|
102
|
+
return self.db.execute("SELECT COUNT(*) FROM docs").fetchone()[0]
|
|
103
|
+
|
|
104
|
+
def close(self) -> None:
|
|
105
|
+
self.db.close()
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Embedding strategy is bring-your-own by default.
|
|
2
|
+
|
|
3
|
+
Pass any callable ``list[str] -> list[list[float]]`` (or an object with an
|
|
4
|
+
``embed`` method) wherever an embedder is accepted. Optional thin wrappers for
|
|
5
|
+
common providers are lazy-imported so the base install stays light.
|
|
6
|
+
|
|
7
|
+
``HashEmbedder`` is a dependency-free, deterministic embedder used by the
|
|
8
|
+
in-memory adapter, tests, and examples so the whole harness runs with no API
|
|
9
|
+
key. It is *not* semantically meaningful — swap in a real provider for quality.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import math
|
|
16
|
+
from typing import Callable, Protocol, Sequence, runtime_checkable
|
|
17
|
+
|
|
18
|
+
from .errors import ConfigurationError, InvalidEmbedderError, MissingDependencyError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@runtime_checkable
|
|
22
|
+
class Embedder(Protocol):
|
|
23
|
+
def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def as_embedder(obj: "Embedder | Callable[[Sequence[str]], list[list[float]]]") -> Embedder:
|
|
27
|
+
"""Coerce a plain callable into the :class:`Embedder` protocol."""
|
|
28
|
+
if isinstance(obj, Embedder):
|
|
29
|
+
return obj
|
|
30
|
+
if callable(obj):
|
|
31
|
+
return _CallableEmbedder(obj)
|
|
32
|
+
raise InvalidEmbedderError(
|
|
33
|
+
"embedder must be an Embedder or a callable(list[str]) -> vectors",
|
|
34
|
+
got=type(obj).__name__,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class _CallableEmbedder:
|
|
39
|
+
def __init__(self, fn: Callable[[Sequence[str]], list[list[float]]]):
|
|
40
|
+
self._fn = fn
|
|
41
|
+
|
|
42
|
+
def embed(self, texts: Sequence[str]) -> list[list[float]]:
|
|
43
|
+
return self._fn(texts)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class HashEmbedder:
|
|
47
|
+
"""Deterministic hashing embedder — zero dependencies, no network.
|
|
48
|
+
|
|
49
|
+
Uses hashed token bucketing (a poor-man's bag-of-words) so that texts
|
|
50
|
+
sharing tokens land near each other. Good enough to exercise the full
|
|
51
|
+
retrieval path in tests and demos.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def __init__(self, dim: int = 256):
|
|
55
|
+
self.dim = dim
|
|
56
|
+
|
|
57
|
+
def embed(self, texts: Sequence[str]) -> list[list[float]]:
|
|
58
|
+
return [self._one(t) for t in texts]
|
|
59
|
+
|
|
60
|
+
def _one(self, text: str) -> list[float]:
|
|
61
|
+
vec = [0.0] * self.dim
|
|
62
|
+
for tok in _tokenize(text):
|
|
63
|
+
h = int.from_bytes(hashlib.blake2b(tok.encode(), digest_size=8).digest(), "big")
|
|
64
|
+
idx = h % self.dim
|
|
65
|
+
sign = 1.0 if (h >> 8) & 1 else -1.0
|
|
66
|
+
vec[idx] += sign
|
|
67
|
+
norm = math.sqrt(sum(v * v for v in vec)) or 1.0
|
|
68
|
+
return [v / norm for v in vec]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _tokenize(text: str) -> list[str]:
|
|
72
|
+
return [t for t in "".join(c.lower() if c.isalnum() else " " for c in text).split() if t]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def get_embedder(provider: str, **kwargs) -> Embedder:
|
|
76
|
+
"""Factory for optional built-in providers (lazy-imported)."""
|
|
77
|
+
provider = provider.lower()
|
|
78
|
+
if provider in ("hash", "test"):
|
|
79
|
+
return HashEmbedder(**kwargs)
|
|
80
|
+
if provider == "openai":
|
|
81
|
+
return _OpenAIEmbedder(**kwargs)
|
|
82
|
+
if provider in ("transformers", "hf", "sentence-transformers"):
|
|
83
|
+
return _TransformersEmbedder(**kwargs)
|
|
84
|
+
raise ConfigurationError("unknown embedding provider", provider=provider)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class _TransformersEmbedder:
|
|
88
|
+
"""Robust loader for HF ``transformers`` embedding models — including custom GTE-v1.5 /
|
|
89
|
+
``model_type="new"`` models whose meta-device init corrupts non-persistent buffers
|
|
90
|
+
(``position_ids``, rotary ``cos_cached``/``sin_cached``) and causes a GPU device-assert
|
|
91
|
+
or NaN outputs. Fixes: load with ``low_cpu_mem_usage=False``, disable the memory-efficient
|
|
92
|
+
``unpad`` attention path, and re-materialize those buffers ON THE MODEL DEVICE.
|
|
93
|
+
|
|
94
|
+
emb = get_embedder("transformers", model="my/custom-gte", pooling="cls")
|
|
95
|
+
vecs = emb.embed(["a query"])
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(self, model: str, device: str | None = None, pooling: str = "cls",
|
|
99
|
+
max_length: int = 512, token: str | None = None, trust_remote_code: bool = True):
|
|
100
|
+
import torch
|
|
101
|
+
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
|
102
|
+
self._torch = torch
|
|
103
|
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
104
|
+
self.pooling = pooling
|
|
105
|
+
self.max_length = max_length
|
|
106
|
+
conf = AutoConfig.from_pretrained(model, trust_remote_code=trust_remote_code, token=token)
|
|
107
|
+
for attr in ("unpad_inputs", "use_memory_efficient_attention"):
|
|
108
|
+
if hasattr(conf, attr):
|
|
109
|
+
setattr(conf, attr, False)
|
|
110
|
+
self.tok = AutoTokenizer.from_pretrained(model, token=token)
|
|
111
|
+
self.model = AutoModel.from_pretrained(
|
|
112
|
+
model, config=conf, trust_remote_code=trust_remote_code,
|
|
113
|
+
low_cpu_mem_usage=False, token=token).to(self.device).eval()
|
|
114
|
+
self._fix_meta_buffers(conf)
|
|
115
|
+
|
|
116
|
+
def _fix_meta_buffers(self, conf) -> None:
|
|
117
|
+
torch, dev = self._torch, self.device
|
|
118
|
+
maxp = getattr(conf, "max_position_embeddings", 512)
|
|
119
|
+
for m in self.model.modules():
|
|
120
|
+
if hasattr(m, "position_ids"):
|
|
121
|
+
m.register_buffer("position_ids", torch.arange(maxp, device=dev), persistent=False)
|
|
122
|
+
if hasattr(m, "cos_cached") and hasattr(m, "inv_freq"):
|
|
123
|
+
L = int(m.cos_cached.shape[0])
|
|
124
|
+
t = torch.arange(L, dtype=torch.float32, device=dev)
|
|
125
|
+
freqs = torch.outer(t, m.inv_freq.float().to(dev))
|
|
126
|
+
emb = torch.cat((freqs, freqs), dim=-1)
|
|
127
|
+
m.register_buffer("cos_cached", emb.cos(), persistent=False)
|
|
128
|
+
m.register_buffer("sin_cached", emb.sin(), persistent=False)
|
|
129
|
+
|
|
130
|
+
def embed(self, texts: Sequence[str]) -> list[list[float]]:
|
|
131
|
+
torch = self._torch
|
|
132
|
+
out = []
|
|
133
|
+
for text in texts:
|
|
134
|
+
enc = self.tok(text, return_tensors="pt", truncation=True, max_length=self.max_length)
|
|
135
|
+
enc = {k: v.to(self.device) for k, v in enc.items()}
|
|
136
|
+
with torch.no_grad():
|
|
137
|
+
h = self.model(**enc).last_hidden_state
|
|
138
|
+
v = h[:, 0] if self.pooling == "cls" else h.mean(dim=1)
|
|
139
|
+
v = torch.nn.functional.normalize(v, p=2, dim=-1)
|
|
140
|
+
out.append(v[0].detach().cpu().tolist())
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class _OpenAIEmbedder:
|
|
145
|
+
def __init__(self, model: str = "text-embedding-3-small", **client_kwargs):
|
|
146
|
+
try:
|
|
147
|
+
from openai import OpenAI
|
|
148
|
+
except ImportError as e: # pragma: no cover - optional dep
|
|
149
|
+
raise MissingDependencyError("openai", extra="search-as-code[providers]") from e
|
|
150
|
+
self._client = OpenAI(**client_kwargs)
|
|
151
|
+
self._model = model
|
|
152
|
+
|
|
153
|
+
def embed(self, texts: Sequence[str]) -> list[list[float]]: # pragma: no cover - network
|
|
154
|
+
resp = self._client.embeddings.create(model=self._model, input=list(texts))
|
|
155
|
+
return [d.embedding for d in resp.data]
|
search_as_code/errors.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Typed exceptions with stable error codes.
|
|
2
|
+
|
|
3
|
+
Every error the SDK raises is a :class:`SacError` carrying a stable string
|
|
4
|
+
``.code`` (e.g. ``"E_INVALID_FILTER"``) for programmatic handling, logging, and
|
|
5
|
+
metrics. Each subclass *also* inherits from the built-in exception that best
|
|
6
|
+
matches its meaning (``ValueError`` / ``TypeError`` / ``RuntimeError`` /
|
|
7
|
+
``ImportError``), so existing ``except ValueError:`` handlers — and code written
|
|
8
|
+
before this hierarchy existed — keep working unchanged.
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
sac.connect("nope")
|
|
12
|
+
except sac.BackendNotFoundError as e:
|
|
13
|
+
log.warning("bad backend", code=e.code, **e.context)
|
|
14
|
+
except ValueError:
|
|
15
|
+
... # still catches it (back-compat)
|
|
16
|
+
|
|
17
|
+
Codes are stable identifiers; the human message may change, the code should not.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SacError(Exception):
|
|
26
|
+
"""Base class for every error raised by search-as-code.
|
|
27
|
+
|
|
28
|
+
Carries a stable ``code`` and an optional structured ``context`` dict so
|
|
29
|
+
callers can branch on the code and log the details without parsing strings.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
code: str = "E_SAC"
|
|
33
|
+
|
|
34
|
+
def __init__(self, message: str = "", **context: Any):
|
|
35
|
+
self.context: dict[str, Any] = context
|
|
36
|
+
if context:
|
|
37
|
+
detail = ", ".join(f"{k}={v!r}" for k, v in context.items())
|
|
38
|
+
message = f"{message} ({detail})" if message else detail
|
|
39
|
+
self.message = message
|
|
40
|
+
super().__init__(message)
|
|
41
|
+
|
|
42
|
+
def __str__(self) -> str: # keep the code visible in tracebacks/logs
|
|
43
|
+
base = super().__str__()
|
|
44
|
+
return f"[{self.code}] {base}" if base else f"[{self.code}]"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ---- configuration / wiring --------------------------------------------------
|
|
48
|
+
class ConfigurationError(SacError, ValueError):
|
|
49
|
+
"""The harness was set up incorrectly (bad backend, provider, options)."""
|
|
50
|
+
|
|
51
|
+
code = "E_CONFIG"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class BackendNotFoundError(ConfigurationError):
|
|
55
|
+
"""``connect(backend)`` was given an unknown backend name."""
|
|
56
|
+
|
|
57
|
+
code = "E_BACKEND_NOT_FOUND"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class MissingDependencyError(SacError, ImportError):
|
|
61
|
+
"""An optional dependency for a backend/provider is not installed."""
|
|
62
|
+
|
|
63
|
+
code = "E_MISSING_DEPENDENCY"
|
|
64
|
+
|
|
65
|
+
def __init__(self, package: str, extra: str | None = None, **context: Any):
|
|
66
|
+
hint = f"pip install '{extra}'" if extra else f"pip install {package}"
|
|
67
|
+
self.package = package
|
|
68
|
+
self.extra = extra
|
|
69
|
+
super().__init__(
|
|
70
|
+
f"optional dependency {package!r} is not installed; {hint}", **context
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ---- bad arguments -----------------------------------------------------------
|
|
75
|
+
class InvalidArgumentError(SacError, ValueError):
|
|
76
|
+
"""A user-supplied argument is out of range or the wrong shape."""
|
|
77
|
+
|
|
78
|
+
code = "E_INVALID_ARGUMENT"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class InvalidModeError(InvalidArgumentError):
|
|
82
|
+
"""An unknown search ``mode`` was requested."""
|
|
83
|
+
|
|
84
|
+
code = "E_INVALID_MODE"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class InvalidFilterError(InvalidArgumentError):
|
|
88
|
+
"""A metadata filter used an unknown operator or malformed structure."""
|
|
89
|
+
|
|
90
|
+
code = "E_INVALID_FILTER"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class DimensionMismatchError(InvalidArgumentError):
|
|
94
|
+
"""A vector's dimensionality does not match the store/embedder's."""
|
|
95
|
+
|
|
96
|
+
code = "E_DIMENSION_MISMATCH"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class InvalidEmbedderError(SacError, TypeError):
|
|
100
|
+
"""``embedder`` is neither an :class:`Embedder` nor a callable."""
|
|
101
|
+
|
|
102
|
+
code = "E_INVALID_EMBEDDER"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# ---- missing pluggable callables ---------------------------------------------
|
|
106
|
+
class GeneratorRequiredError(SacError, RuntimeError):
|
|
107
|
+
"""A query-side primitive needs an LLM ``generator`` but none was supplied."""
|
|
108
|
+
|
|
109
|
+
code = "E_GENERATOR_REQUIRED"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class ExtractorRequiredError(SacError, RuntimeError):
|
|
113
|
+
"""``extract()`` needs an ``extractor`` callable but none was supplied."""
|
|
114
|
+
|
|
115
|
+
code = "E_EXTRACTOR_REQUIRED"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---- runtime / backend failures ----------------------------------------------
|
|
119
|
+
class EmbeddingError(SacError, RuntimeError):
|
|
120
|
+
"""The embedder failed or returned a malformed result."""
|
|
121
|
+
|
|
122
|
+
code = "E_EMBEDDING"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class BackendError(SacError, RuntimeError):
|
|
126
|
+
"""A backend call failed (after any retries) — network, timeout, or server."""
|
|
127
|
+
|
|
128
|
+
code = "E_BACKEND"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
__all__ = [
|
|
132
|
+
"SacError",
|
|
133
|
+
"ConfigurationError",
|
|
134
|
+
"BackendNotFoundError",
|
|
135
|
+
"MissingDependencyError",
|
|
136
|
+
"InvalidArgumentError",
|
|
137
|
+
"InvalidModeError",
|
|
138
|
+
"InvalidFilterError",
|
|
139
|
+
"DimensionMismatchError",
|
|
140
|
+
"InvalidEmbedderError",
|
|
141
|
+
"GeneratorRequiredError",
|
|
142
|
+
"ExtractorRequiredError",
|
|
143
|
+
"EmbeddingError",
|
|
144
|
+
"BackendError",
|
|
145
|
+
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""``sac.explore`` — the corpus **exploration / onboarding phase**.
|
|
2
|
+
|
|
3
|
+
Run once when SAC is installed against a corpus. It samples the data, profiles it
|
|
4
|
+
(schema + content-type mix + LLM characterization), and — as later stages land —
|
|
5
|
+
induces a domain ontology, generates synthetic queries, trains a primitive router,
|
|
6
|
+
and mines templates/few-shots. Everything it learns is written to a versioned
|
|
7
|
+
:class:`ProfilePack` that a ``Session`` loads at query time.
|
|
8
|
+
|
|
9
|
+
import search_as_code as sac
|
|
10
|
+
s = sac.Session("opensearch", index="docs", ...)
|
|
11
|
+
pack = sac.explore(s, out="docs_pack/") # onboarding
|
|
12
|
+
print(pack.report())
|
|
13
|
+
|
|
14
|
+
The pipeline is resumable (each stage writes its own artifact), drift-aware (re-runs
|
|
15
|
+
when the corpus fingerprint changes), and validate-before-keep (a tuning is kept only
|
|
16
|
+
if it beats baseline).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from .engine import ExploreContext, Stage, corpus_fingerprint, default_pipeline, explore
|
|
20
|
+
from .pack import ProfilePack
|
|
21
|
+
from .report import write_csv_report
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"explore",
|
|
25
|
+
"ProfilePack",
|
|
26
|
+
"Stage",
|
|
27
|
+
"ExploreContext",
|
|
28
|
+
"default_pipeline",
|
|
29
|
+
"corpus_fingerprint",
|
|
30
|
+
"write_csv_report",
|
|
31
|
+
]
|