kosha-client 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kosha_client-0.1.0/PKG-INFO +9 -0
- kosha_client-0.1.0/kosha_client/__init__.py +20 -0
- kosha_client-0.1.0/kosha_client/client.py +806 -0
- kosha_client-0.1.0/kosha_client.egg-info/PKG-INFO +9 -0
- kosha_client-0.1.0/kosha_client.egg-info/SOURCES.txt +7 -0
- kosha_client-0.1.0/kosha_client.egg-info/dependency_links.txt +1 -0
- kosha_client-0.1.0/kosha_client.egg-info/top_level.txt +2 -0
- kosha_client-0.1.0/pyproject.toml +18 -0
- kosha_client-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kosha-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: OpenSearch-compatible Python client for the Kosha search engine
|
|
5
|
+
Author-email: DecoverAI <engineering@decoverai.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/ravi-decoverai/Kosha
|
|
8
|
+
Project-URL: Source, https://github.com/ravi-decoverai/Kosha
|
|
9
|
+
Requires-Python: >=3.10
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Kosha client — a drop-in replacement for opensearchpy.OpenSearch.
|
|
2
|
+
|
|
3
|
+
Usage (swap two lines in Sage's connection_factory.py)::
|
|
4
|
+
|
|
5
|
+
# Before:
|
|
6
|
+
# from opensearchpy import OpenSearch
|
|
7
|
+
# client = OpenSearch(hosts=[...], http_auth=(...))
|
|
8
|
+
|
|
9
|
+
# After:
|
|
10
|
+
from kosha_client import KoshaClient as OpenSearch
|
|
11
|
+
client = OpenSearch(hosts=[...], http_auth=(...))
|
|
12
|
+
|
|
13
|
+
All existing opensearch_dsl code (Search, Q, Document.save, helpers.bulk)
|
|
14
|
+
works because KoshaClient.search(), .index(), .bulk() return dicts in the
|
|
15
|
+
same shape as the OpenSearch / Elasticsearch JSON API.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from .client import KoshaClient
|
|
19
|
+
|
|
20
|
+
__all__ = ["KoshaClient"]
|
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
"""KoshaClient — an OpenSearch-compatible client that talks to Kosha."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import time
|
|
8
|
+
import urllib.parse
|
|
9
|
+
import urllib.request
|
|
10
|
+
from typing import Any, Sequence
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
# ─── Public interface ──────────────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
class KoshaClient:
|
|
17
|
+
"""Drop-in replacement for ``opensearchpy.OpenSearch``.
|
|
18
|
+
|
|
19
|
+
Translates the most common OpenSearch / Elasticsearch operations into
|
|
20
|
+
Kosha HTTP API calls and returns response dicts in the same shape as
|
|
21
|
+
the OpenSearch JSON API so that ``opensearch_dsl`` wrappers
|
|
22
|
+
(``Search``, ``Q``, ``Document.save``, ``helpers.bulk``) work unchanged.
|
|
23
|
+
|
|
24
|
+
Phase 1 focus — BM25 lexical search only:
|
|
25
|
+
|
|
26
|
+
* ``search()`` — ``{"query": {"match": ...}}``,
|
|
27
|
+
``{"query": {"bool": {"must": [...], "filter": [...]}}}``
|
|
28
|
+
* ``index()`` — single-document index
|
|
29
|
+
* ``bulk()`` — multi-document index via ``opensearchpy.helpers.bulk``
|
|
30
|
+
* ``count()`` — document count
|
|
31
|
+
* ``update()`` — update by id
|
|
32
|
+
* ``delete_by_query()`` — delete matching documents
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
hosts: Any = None,
|
|
38
|
+
http_auth: Any = None,
|
|
39
|
+
timeout: int = 60,
|
|
40
|
+
max_retries: int = 3,
|
|
41
|
+
retry_on_timeout: bool = True,
|
|
42
|
+
pool_maxsize: int = 20,
|
|
43
|
+
**kwargs: Any,
|
|
44
|
+
) -> None:
|
|
45
|
+
# Normalise the Kosha base URL from the same ``hosts`` format
|
|
46
|
+
# that ``opensearchpy`` accepts.
|
|
47
|
+
if isinstance(hosts, str):
|
|
48
|
+
self._kosha_url = hosts.rstrip("/")
|
|
49
|
+
elif isinstance(hosts, (list, tuple)) and len(hosts) > 0:
|
|
50
|
+
h = hosts[0]
|
|
51
|
+
if isinstance(h, str):
|
|
52
|
+
self._kosha_url = h.rstrip("/")
|
|
53
|
+
elif isinstance(h, dict):
|
|
54
|
+
scheme = h.get("scheme", "http")
|
|
55
|
+
host = h.get("host", "localhost")
|
|
56
|
+
port = h.get("port", 8080)
|
|
57
|
+
self._kosha_url = f"{scheme}://{host}:{port}"
|
|
58
|
+
else:
|
|
59
|
+
self._kosha_url = "http://localhost:8080"
|
|
60
|
+
else:
|
|
61
|
+
self._kosha_url = kwargs.get("kosha_url", "http://localhost:8080")
|
|
62
|
+
|
|
63
|
+
self._timeout = timeout
|
|
64
|
+
self._auth = http_auth
|
|
65
|
+
|
|
66
|
+
# Kosha namespace → index name mapping.
|
|
67
|
+
# In Phase 1, index name is used directly as the namespace.
|
|
68
|
+
self._namespace = kwargs.get("namespace", "default")
|
|
69
|
+
|
|
70
|
+
logger.info("KoshaClient targeting %s namespace=%s", self._kosha_url, self._namespace)
|
|
71
|
+
|
|
72
|
+
# ── Low-level request helpers ──────────────────────────────────────────
|
|
73
|
+
|
|
74
|
+
def _request(self, method: str, path: str, body: Any = None) -> Any:
|
|
75
|
+
url = f"{self._kosha_url}/{path.lstrip('/')}"
|
|
76
|
+
data = json.dumps(body).encode() if body is not None else None
|
|
77
|
+
req = urllib.request.Request(url, data=data, method=method)
|
|
78
|
+
req.add_header("Content-Type", "application/json")
|
|
79
|
+
if self._auth:
|
|
80
|
+
import base64
|
|
81
|
+
user, pwd = self._auth
|
|
82
|
+
token = base64.b64encode(f"{user}:{pwd}".encode()).decode()
|
|
83
|
+
req.add_header("Authorization", f"Basic {token}")
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
resp = urllib.request.urlopen(req, timeout=self._timeout)
|
|
87
|
+
return json.loads(resp.read().decode())
|
|
88
|
+
except urllib.error.HTTPError as e:
|
|
89
|
+
body_bytes = e.read()
|
|
90
|
+
try:
|
|
91
|
+
err = json.loads(body_bytes.decode())
|
|
92
|
+
except json.JSONDecodeError:
|
|
93
|
+
err = {"error": body_bytes.decode()}
|
|
94
|
+
logger.warning("Kosha request failed: %s %s → %s %s", method, path, e.code, err)
|
|
95
|
+
raise KoshaRequestError(e.code, err.get("error", str(e)), err) from e
|
|
96
|
+
|
|
97
|
+
# ── Search ─────────────────────────────────────────────────────────────
|
|
98
|
+
|
|
99
|
+
def search(self, index: str | None = None, body: dict | None = None, **params: Any) -> dict:
|
|
100
|
+
ns = index or self._namespace
|
|
101
|
+
|
|
102
|
+
# kNN/semantic not supported in Phase 1 — return empty.
|
|
103
|
+
if body and ("knn" in body or "knn" in (body.get("query") or {})):
|
|
104
|
+
return self._build_search_response([], 0, body.get("size", 10), 0, 0)
|
|
105
|
+
|
|
106
|
+
query_text = self._extract_query_text(body) if body else ""
|
|
107
|
+
size = body.get("size", 10) if body else 10
|
|
108
|
+
from_ = body.get("from", 0) if body else 0
|
|
109
|
+
filter_clause = self._extract_filter(body)
|
|
110
|
+
aggs = self._extract_aggs(body)
|
|
111
|
+
wildcard = self._extract_wildcard(body)
|
|
112
|
+
match_phrase = self._extract_match_phrase(body)
|
|
113
|
+
|
|
114
|
+
# Determine if we need POST (agg/wildcard/phrase/filter present).
|
|
115
|
+
needs_post = bool(filter_clause or aggs or wildcard or match_phrase)
|
|
116
|
+
|
|
117
|
+
if not needs_post:
|
|
118
|
+
bm25_params = {}
|
|
119
|
+
q = body and (body.get("query") or {})
|
|
120
|
+
if q:
|
|
121
|
+
bm25_params = self._extract_bm25_params(q)
|
|
122
|
+
query_params = {"ns": ns, "q": query_text, "max_results": str(size + from_)}
|
|
123
|
+
if bm25_params:
|
|
124
|
+
query_params.update(bm25_params)
|
|
125
|
+
url = f"{self._kosha_url}/search?{urllib.parse.urlencode(query_params)}"
|
|
126
|
+
try:
|
|
127
|
+
req = urllib.request.Request(url)
|
|
128
|
+
if self._auth:
|
|
129
|
+
import base64
|
|
130
|
+
user, pwd = self._auth
|
|
131
|
+
token = base64.b64encode(f"{user}:{pwd}".encode()).decode()
|
|
132
|
+
req.add_header("Authorization", f"Basic {token}")
|
|
133
|
+
start = time.monotonic()
|
|
134
|
+
resp = urllib.request.urlopen(req, timeout=self._timeout)
|
|
135
|
+
took_ms = int((time.monotonic() - start) * 1000)
|
|
136
|
+
result = json.loads(resp.read().decode())
|
|
137
|
+
except urllib.error.HTTPError as e:
|
|
138
|
+
took_ms = 0
|
|
139
|
+
if e.code == 404:
|
|
140
|
+
return self._build_search_response([], from_, size, took_ms)
|
|
141
|
+
body_bytes = e.read()
|
|
142
|
+
try:
|
|
143
|
+
err = json.loads(body_bytes.decode())
|
|
144
|
+
except json.JSONDecodeError:
|
|
145
|
+
err = {"error": body_bytes.decode()}
|
|
146
|
+
raise KoshaRequestError(e.code, err.get("error", str(e)), err) from e
|
|
147
|
+
kosha_hits = result.get("results", [])
|
|
148
|
+
total = result.get("total_hits", 0)
|
|
149
|
+
return self._build_search_response(kosha_hits, from_, size, took_ms, total)
|
|
150
|
+
|
|
151
|
+
kosha_body = {
|
|
152
|
+
"namespace": ns,
|
|
153
|
+
"query_text": query_text,
|
|
154
|
+
"max_results": size + from_,
|
|
155
|
+
"from": from_,
|
|
156
|
+
}
|
|
157
|
+
if filter_clause:
|
|
158
|
+
kosha_body["filter"] = filter_clause
|
|
159
|
+
if aggs:
|
|
160
|
+
kosha_body["aggs"] = aggs
|
|
161
|
+
if wildcard:
|
|
162
|
+
kosha_body["wildcard"] = wildcard
|
|
163
|
+
if match_phrase:
|
|
164
|
+
kosha_body["match_phrase"] = match_phrase
|
|
165
|
+
|
|
166
|
+
result = self._request("POST", "search", body=kosha_body)
|
|
167
|
+
kosha_hits = result.get("results", [])
|
|
168
|
+
total = result.get("total_hits", 0)
|
|
169
|
+
kosha_aggs = result.get("aggregations")
|
|
170
|
+
|
|
171
|
+
response = self._build_search_response(kosha_hits, 0, size, 0, total)
|
|
172
|
+
if kosha_aggs:
|
|
173
|
+
response["aggregations"] = kosha_aggs
|
|
174
|
+
return response
|
|
175
|
+
|
|
176
|
+
def _build_search_response(
|
|
177
|
+
self,
|
|
178
|
+
kosha_hits: list[dict],
|
|
179
|
+
from_: int,
|
|
180
|
+
size: int,
|
|
181
|
+
took_ms: int,
|
|
182
|
+
total: int | None = None,
|
|
183
|
+
) -> dict:
|
|
184
|
+
if total is None:
|
|
185
|
+
total = len(kosha_hits)
|
|
186
|
+
|
|
187
|
+
hits = []
|
|
188
|
+
for k_hit in kosha_hits:
|
|
189
|
+
doc_id = k_hit.get("doc_id", "")
|
|
190
|
+
score = k_hit.get("score", 0.0)
|
|
191
|
+
source = {}
|
|
192
|
+
for field in k_hit.get("fields", []):
|
|
193
|
+
source[field["name"]] = field.get("value", "")
|
|
194
|
+
# Store field type for filter-aware operations.
|
|
195
|
+
if field.get("field_type") not in ("Text", None):
|
|
196
|
+
source[f"__type__{field['name']}"] = field["field_type"]
|
|
197
|
+
hits.append({
|
|
198
|
+
"_index": self._namespace,
|
|
199
|
+
"_id": doc_id,
|
|
200
|
+
"_score": score,
|
|
201
|
+
"_source": source,
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
# Apply offset/pagination in Python (Kosha returns flat top-N).
|
|
205
|
+
page = hits[from_: from_ + size]
|
|
206
|
+
|
|
207
|
+
return {
|
|
208
|
+
"took": took_ms,
|
|
209
|
+
"timed_out": False,
|
|
210
|
+
"_shards": {"total": 1, "successful": 1, "skipped": 0, "failed": 0},
|
|
211
|
+
"hits": {
|
|
212
|
+
"total": {"value": total, "relation": "eq"},
|
|
213
|
+
"max_score": max((h["_score"] for h in page), default=0.0),
|
|
214
|
+
"hits": page,
|
|
215
|
+
},
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
@staticmethod
|
|
219
|
+
def _extract_query_text(body: dict | None) -> str:
|
|
220
|
+
"""Extract the user's query text from an OpenSearch query body."""
|
|
221
|
+
if not body:
|
|
222
|
+
return ""
|
|
223
|
+
query = body.get("query") or {}
|
|
224
|
+
return KoshaClient._extract_from_query_dsl(query)
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
def _extract_from_query_dsl(query: dict) -> str:
|
|
228
|
+
"""Recursively extract search terms from query DSL."""
|
|
229
|
+
if not query:
|
|
230
|
+
return ""
|
|
231
|
+
|
|
232
|
+
# match: {"match": {"field": "text"}} or {"match": {"field": {"query": "text"}}}
|
|
233
|
+
match = query.get("match")
|
|
234
|
+
if match is not None:
|
|
235
|
+
for field, val in match.items():
|
|
236
|
+
if isinstance(val, str):
|
|
237
|
+
return val
|
|
238
|
+
if isinstance(val, dict) and "query" in val:
|
|
239
|
+
return val["query"]
|
|
240
|
+
return ""
|
|
241
|
+
|
|
242
|
+
# multi_match: {"multi_match": {"query": "text", "fields": [...]}}
|
|
243
|
+
multi_match = query.get("multi_match")
|
|
244
|
+
if multi_match is not None:
|
|
245
|
+
return multi_match.get("query", "")
|
|
246
|
+
|
|
247
|
+
# bool: {"bool": {"must": [...], "should": [...], "filter": [...]}}
|
|
248
|
+
# Only extract text from must/should (full-text clauses).
|
|
249
|
+
# Filter clauses (term, terms, range) are not query text.
|
|
250
|
+
bool_q = query.get("bool")
|
|
251
|
+
if bool_q is not None:
|
|
252
|
+
texts = []
|
|
253
|
+
for clause_key in ("must", "should"):
|
|
254
|
+
for clause in bool_q.get(clause_key, []):
|
|
255
|
+
if not KoshaClient._is_filter_only_clause(clause):
|
|
256
|
+
t = KoshaClient._extract_from_query_dsl(clause)
|
|
257
|
+
if t:
|
|
258
|
+
texts.append(t)
|
|
259
|
+
return " ".join(texts)
|
|
260
|
+
|
|
261
|
+
# term: {"term": {"field": "value"}} — exact match, not full-text.
|
|
262
|
+
# We return empty because Kosha does BM25, not term queries (Phase 1).
|
|
263
|
+
term = query.get("term")
|
|
264
|
+
if term is not None:
|
|
265
|
+
for field, val in term.items():
|
|
266
|
+
if isinstance(val, str):
|
|
267
|
+
return val
|
|
268
|
+
return ""
|
|
269
|
+
|
|
270
|
+
# match_all
|
|
271
|
+
if query.get("match_all") is not None:
|
|
272
|
+
return ""
|
|
273
|
+
|
|
274
|
+
# function_score: {"function_score": {"query": {...}, ...}}
|
|
275
|
+
fn_score = query.get("function_score")
|
|
276
|
+
if fn_score is not None:
|
|
277
|
+
return KoshaClient._extract_from_query_dsl(fn_score.get("query", {}))
|
|
278
|
+
|
|
279
|
+
return ""
|
|
280
|
+
|
|
281
|
+
@staticmethod
|
|
282
|
+
def _extract_bm25_params(query: dict) -> dict:
|
|
283
|
+
"""Extract BM25 tuning parameters if present in the query body."""
|
|
284
|
+
params = {}
|
|
285
|
+
if "settings" in query:
|
|
286
|
+
sim = query["settings"].get("index", {}).get("similarity", {}).get("default", {})
|
|
287
|
+
if sim.get("type") == "BM25":
|
|
288
|
+
k1 = sim.get("k1", 1.2)
|
|
289
|
+
b = sim.get("b", 0.75)
|
|
290
|
+
params["k1"] = str(k1)
|
|
291
|
+
params["b"] = str(b)
|
|
292
|
+
return params
|
|
293
|
+
|
|
294
|
+
@staticmethod
|
|
295
|
+
def _extract_filter(body: dict | None) -> dict | None:
|
|
296
|
+
"""Extract a filter clause from an OpenSearch query body.
|
|
297
|
+
|
|
298
|
+
Handles:
|
|
299
|
+
- ``body["query"]["bool"]["filter"]`` — ES bool filter clauses
|
|
300
|
+
- ``body["post_filter"]`` — ES post_filter
|
|
301
|
+
- ``body["query"]["bool"]["must_not"]`` — ES bool must_not
|
|
302
|
+
"""
|
|
303
|
+
if not body:
|
|
304
|
+
return None
|
|
305
|
+
|
|
306
|
+
# Collect filter clauses from various locations.
|
|
307
|
+
must_clauses: list = []
|
|
308
|
+
must_not_clauses: list = []
|
|
309
|
+
should_clauses: list = []
|
|
310
|
+
has_clauses = False
|
|
311
|
+
|
|
312
|
+
query = body.get("query") or {}
|
|
313
|
+
|
|
314
|
+
# bool.filter clauses
|
|
315
|
+
bool_q = query.get("bool")
|
|
316
|
+
if bool_q:
|
|
317
|
+
for clause in bool_q.get("filter", []):
|
|
318
|
+
translated = KoshaClient._translate_es_clause(clause)
|
|
319
|
+
if translated:
|
|
320
|
+
must_clauses.append(translated)
|
|
321
|
+
has_clauses = True
|
|
322
|
+
for clause in bool_q.get("must_not", []):
|
|
323
|
+
translated = KoshaClient._translate_es_clause(clause)
|
|
324
|
+
if translated:
|
|
325
|
+
must_not_clauses.append(translated)
|
|
326
|
+
has_clauses = True
|
|
327
|
+
for clause in bool_q.get("must", []):
|
|
328
|
+
# Only translate term/terms/range in must clauses
|
|
329
|
+
# (match clauses are handled by _extract_query_text).
|
|
330
|
+
translated = KoshaClient._translate_es_clause(clause)
|
|
331
|
+
if translated and KoshaClient._is_filter_only_clause(clause):
|
|
332
|
+
must_clauses.append(translated)
|
|
333
|
+
has_clauses = True
|
|
334
|
+
|
|
335
|
+
# post_filter
|
|
336
|
+
post_filter = body.get("post_filter")
|
|
337
|
+
if post_filter:
|
|
338
|
+
translated = KoshaClient._translate_es_clause(post_filter)
|
|
339
|
+
if translated:
|
|
340
|
+
must_clauses.append(translated)
|
|
341
|
+
has_clauses = True
|
|
342
|
+
|
|
343
|
+
if not has_clauses:
|
|
344
|
+
return None
|
|
345
|
+
|
|
346
|
+
result: dict = {}
|
|
347
|
+
if must_clauses:
|
|
348
|
+
result["bool"] = result.get("bool", {})
|
|
349
|
+
result["bool"]["must"] = must_clauses
|
|
350
|
+
if must_not_clauses:
|
|
351
|
+
result["bool"] = result.get("bool", {})
|
|
352
|
+
result["bool"]["must_not"] = must_not_clauses
|
|
353
|
+
|
|
354
|
+
return result if result else None
|
|
355
|
+
|
|
356
|
+
@staticmethod
|
|
357
|
+
def _is_filter_only_clause(clause: dict) -> bool:
|
|
358
|
+
"""Check if a clause is a filter-only clause (not a match query)."""
|
|
359
|
+
if not clause:
|
|
360
|
+
return False
|
|
361
|
+
if "term" in clause or "terms" in clause or "range" in clause:
|
|
362
|
+
return True
|
|
363
|
+
if "exists" in clause or "prefix" in clause or "wildcard" in clause:
|
|
364
|
+
return True
|
|
365
|
+
if "match_all" in clause:
|
|
366
|
+
return True
|
|
367
|
+
return False
|
|
368
|
+
|
|
369
|
+
@staticmethod
|
|
370
|
+
def _translate_es_clause(clause: dict) -> dict | None:
|
|
371
|
+
"""Translate a single ES filter clause to Kosha format."""
|
|
372
|
+
if not clause:
|
|
373
|
+
return None
|
|
374
|
+
|
|
375
|
+
# term: {"term": {"matterId": "value"}}
|
|
376
|
+
if "term" in clause:
|
|
377
|
+
return clause # Kosha accepts the same format
|
|
378
|
+
|
|
379
|
+
# terms: {"terms": {"documentId": ["v1", "v2"]}}
|
|
380
|
+
if "terms" in clause:
|
|
381
|
+
return clause # Kosha accepts the same format
|
|
382
|
+
|
|
383
|
+
# range: {"range": {"sentAt": {"gte": "...", "lte": "..."}}}
|
|
384
|
+
if "range" in clause:
|
|
385
|
+
# Normalize range values to strings for Kosha.
|
|
386
|
+
bounds = {}
|
|
387
|
+
for field, val in clause["range"].items():
|
|
388
|
+
if isinstance(val, dict):
|
|
389
|
+
bounds[field] = {k: str(v) for k, v in val.items()}
|
|
390
|
+
else:
|
|
391
|
+
bounds[field] = val
|
|
392
|
+
return {"range": bounds}
|
|
393
|
+
|
|
394
|
+
# bool: nested bool (recursive)
|
|
395
|
+
if "bool" in clause:
|
|
396
|
+
return clause # Kosha accepts the same format
|
|
397
|
+
|
|
398
|
+
# match_all: {"match_all": {}}
|
|
399
|
+
if "match_all" in clause:
|
|
400
|
+
return clause
|
|
401
|
+
|
|
402
|
+
# exists: {"exists": {"field": "..."}}
|
|
403
|
+
if "exists" in clause:
|
|
404
|
+
# Phase 1: skip exists (not supported yet).
|
|
405
|
+
return None
|
|
406
|
+
|
|
407
|
+
return None
|
|
408
|
+
|
|
409
|
+
# ── Index ──────────────────────────────────────────────────────────────
|
|
410
|
+
|
|
411
|
+
@staticmethod
|
|
412
|
+
def _extract_aggs(body: dict | None) -> dict | None:
|
|
413
|
+
"""Extract aggregations from an ES query body."""
|
|
414
|
+
if not body:
|
|
415
|
+
return None
|
|
416
|
+
aggs = body.get("aggs") or body.get("aggregations")
|
|
417
|
+
if aggs:
|
|
418
|
+
return aggs # Kosha accepts the same format as ES
|
|
419
|
+
return None
|
|
420
|
+
|
|
421
|
+
@staticmethod
|
|
422
|
+
def _extract_wildcard(body: dict | None) -> dict | None:
|
|
423
|
+
"""Extract wildcard query from an ES query body.
|
|
424
|
+
Converts ES format to Kosha format.
|
|
425
|
+
"""
|
|
426
|
+
if not body:
|
|
427
|
+
return None
|
|
428
|
+
query = body.get("query") or {}
|
|
429
|
+
wc = query.get("wildcard")
|
|
430
|
+
if not wc:
|
|
431
|
+
return None
|
|
432
|
+
# ES: {"wildcard": {"field": {"value": "*Smith*", "case_insensitive": true}}}
|
|
433
|
+
for field, spec in wc.items():
|
|
434
|
+
if isinstance(spec, dict):
|
|
435
|
+
return {
|
|
436
|
+
"field": field,
|
|
437
|
+
"pattern": spec.get("value", spec.get("wildcard", "")),
|
|
438
|
+
"case_insensitive": spec.get("case_insensitive", True),
|
|
439
|
+
}
|
|
440
|
+
return {
|
|
441
|
+
"field": field,
|
|
442
|
+
"pattern": spec,
|
|
443
|
+
"case_insensitive": True,
|
|
444
|
+
}
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
@staticmethod
|
|
448
|
+
def _extract_match_phrase(body: dict | None) -> dict | None:
|
|
449
|
+
"""Extract match_phrase query from an ES query body.
|
|
450
|
+
Converts ES format to Kosha format.
|
|
451
|
+
"""
|
|
452
|
+
if not body:
|
|
453
|
+
return None
|
|
454
|
+
query = body.get("query") or {}
|
|
455
|
+
mp = query.get("match_phrase")
|
|
456
|
+
if not mp:
|
|
457
|
+
return None
|
|
458
|
+
# ES: {"match_phrase": {"field": {"query": "phrase text", "slop": 2}}}
|
|
459
|
+
for field, spec in mp.items():
|
|
460
|
+
if isinstance(spec, dict):
|
|
461
|
+
return {
|
|
462
|
+
"field": field,
|
|
463
|
+
"phrase": spec.get("query", ""),
|
|
464
|
+
"slop": spec.get("slop", 0),
|
|
465
|
+
}
|
|
466
|
+
return {
|
|
467
|
+
"field": field,
|
|
468
|
+
"phrase": spec,
|
|
469
|
+
"slop": 0,
|
|
470
|
+
}
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
@staticmethod
|
|
474
|
+
def _field_to_kosha(name: str, value: str, field_type: str = "Text") -> dict:
|
|
475
|
+
return {"name": name, "field_type": field_type, "value": value}
|
|
476
|
+
|
|
477
|
+
def index(self, index: str | None = None, id: str | None = None,
|
|
478
|
+
body: dict | None = None, **params: Any) -> dict:
|
|
479
|
+
"""Index a single document."""
|
|
480
|
+
ns = index or self._namespace
|
|
481
|
+
fields = []
|
|
482
|
+
for k, v in (body or {}).items():
|
|
483
|
+
if isinstance(v, str):
|
|
484
|
+
fields.append(self._field_to_kosha(k, v, "Text"))
|
|
485
|
+
elif isinstance(v, bool):
|
|
486
|
+
fields.append(self._field_to_kosha(k, str(v).lower(), "Boolean"))
|
|
487
|
+
elif isinstance(v, (int, float)):
|
|
488
|
+
fields.append(self._field_to_kosha(k, str(v), "Float"))
|
|
489
|
+
doc = {
|
|
490
|
+
"id": id or "",
|
|
491
|
+
"fields": fields,
|
|
492
|
+
}
|
|
493
|
+
payload = {"namespace": ns, "documents": [doc]}
|
|
494
|
+
result = self._request("POST", "index", body=payload)
|
|
495
|
+
return {
|
|
496
|
+
"_index": ns,
|
|
497
|
+
"_id": id or "",
|
|
498
|
+
"_version": 1,
|
|
499
|
+
"result": "created",
|
|
500
|
+
"_shards": {"total": 1, "successful": 1, "failed": 0},
|
|
501
|
+
"_seq_no": result.get("indexed_count", 1),
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
# ── Bulk ───────────────────────────────────────────────────────────────
|
|
505
|
+
|
|
506
|
+
def bulk(self, body: Sequence | str, index: str | None = None,
|
|
507
|
+
**params: Any) -> dict:
|
|
508
|
+
"""Bulk-index documents.
|
|
509
|
+
|
|
510
|
+
Accepts the standard OpenSearch bulk body format (action+source lines
|
|
511
|
+
as a list or newline-delimited string) and indexes each document via
|
|
512
|
+
Kosha.
|
|
513
|
+
"""
|
|
514
|
+
ns = index or self._namespace
|
|
515
|
+
documents: list[dict] = []
|
|
516
|
+
errors: list[dict] = []
|
|
517
|
+
|
|
518
|
+
# Parse the bulk body.
|
|
519
|
+
lines = body if isinstance(body, list) else body.strip().split("\n")
|
|
520
|
+
i = 0
|
|
521
|
+
while i < len(lines):
|
|
522
|
+
action_line = lines[i]
|
|
523
|
+
try:
|
|
524
|
+
action = json.loads(action_line) if isinstance(action_line, str) else action_line
|
|
525
|
+
except json.JSONDecodeError:
|
|
526
|
+
i += 1
|
|
527
|
+
continue
|
|
528
|
+
op_type = None
|
|
529
|
+
doc_id = None
|
|
530
|
+
doc_index = None
|
|
531
|
+
for key in ("index", "create", "update", "delete"):
|
|
532
|
+
if key in action:
|
|
533
|
+
op_type = key
|
|
534
|
+
meta = action[key] or {}
|
|
535
|
+
doc_id = meta.get("_id")
|
|
536
|
+
doc_index = meta.get("_index", ns)
|
|
537
|
+
break
|
|
538
|
+
|
|
539
|
+
if op_type == "delete":
|
|
540
|
+
errors.append({"delete": {"_id": doc_id, "status": 501, "error": "not implemented"}})
|
|
541
|
+
i += 1
|
|
542
|
+
continue
|
|
543
|
+
|
|
544
|
+
if i + 1 >= len(lines):
|
|
545
|
+
break
|
|
546
|
+
source_line = lines[i + 1]
|
|
547
|
+
try:
|
|
548
|
+
source = json.loads(source_line) if isinstance(source_line, str) else source_line
|
|
549
|
+
except json.JSONDecodeError:
|
|
550
|
+
source = {}
|
|
551
|
+
i += 2
|
|
552
|
+
|
|
553
|
+
fields = []
|
|
554
|
+
for k, v in source.items():
|
|
555
|
+
if isinstance(v, str):
|
|
556
|
+
fields.append(self._field_to_kosha(k, v, "Text"))
|
|
557
|
+
elif isinstance(v, bool):
|
|
558
|
+
fields.append(self._field_to_kosha(k, str(v).lower(), "Boolean"))
|
|
559
|
+
elif isinstance(v, (int, float)):
|
|
560
|
+
fields.append(self._field_to_kosha(k, str(v), "Float"))
|
|
561
|
+
doc = {
|
|
562
|
+
"id": doc_id or "",
|
|
563
|
+
"fields": fields,
|
|
564
|
+
}
|
|
565
|
+
documents.append(doc)
|
|
566
|
+
|
|
567
|
+
if documents:
|
|
568
|
+
payload = {"namespace": ns, "documents": documents}
|
|
569
|
+
self._request("POST", "index", body=payload)
|
|
570
|
+
|
|
571
|
+
return {
|
|
572
|
+
"errors": bool(errors),
|
|
573
|
+
"items": (
|
|
574
|
+
[{"index": {"_index": ns, "_id": d["id"], "status": 201}}
|
|
575
|
+
for d in documents]
|
|
576
|
+
+ errors
|
|
577
|
+
),
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
# ── Count ──────────────────────────────────────────────────────────────
|
|
581
|
+
|
|
582
|
+
def count(self, index: str | None = None, body: dict | None = None, **params: Any) -> dict:
|
|
583
|
+
"""Return document count from a broad search.
|
|
584
|
+
|
|
585
|
+
Kosha does not have a dedicated count API, so we search with a
|
|
586
|
+
non-empty placeholder (``*`` rendered as ``match``) and report the
|
|
587
|
+
total_hits.
|
|
588
|
+
"""
|
|
589
|
+
ns = index or self._namespace
|
|
590
|
+
try:
|
|
591
|
+
result = self.search(index=ns, body={"query": {"match": {"_all": "*"}}, "size": 0})
|
|
592
|
+
count = result["hits"]["total"]["value"]
|
|
593
|
+
except KoshaRequestError:
|
|
594
|
+
count = 0
|
|
595
|
+
return {"count": count, "_shards": {"total": 1, "successful": 1, "failed": 0}}
|
|
596
|
+
|
|
597
|
+
# ── Update ─────────────────────────────────────────────────────────────
|
|
598
|
+
|
|
599
|
+
def update(self, index: str | None = None, id: str | None = None,
|
|
600
|
+
body: dict | None = None, **params: Any) -> dict:
|
|
601
|
+
"""Update a document by re-indexing (tombstone-based in Phase 1)."""
|
|
602
|
+
ns = index or self._namespace
|
|
603
|
+
doc_body = (body or {}).get("doc", body or {})
|
|
604
|
+
fields = []
|
|
605
|
+
for k, v in doc_body.items():
|
|
606
|
+
if isinstance(v, str):
|
|
607
|
+
fields.append(self._field_to_kosha(k, v, "Text"))
|
|
608
|
+
elif isinstance(v, bool):
|
|
609
|
+
fields.append(self._field_to_kosha(k, str(v).lower(), "Boolean"))
|
|
610
|
+
elif isinstance(v, (int, float)):
|
|
611
|
+
fields.append(self._field_to_kosha(k, str(v), "Float"))
|
|
612
|
+
payload = {"namespace": ns, "documents": [{"id": id or "", "fields": fields}]}
|
|
613
|
+
self._request("POST", "index", body=payload)
|
|
614
|
+
return {
|
|
615
|
+
"_index": ns,
|
|
616
|
+
"_id": id or "",
|
|
617
|
+
"_version": 2,
|
|
618
|
+
"result": "updated",
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
# ── Delete by query ────────────────────────────────────────────────────
|
|
622
|
+
|
|
623
|
+
def delete_by_query(self, index: str | None = None,
|
|
624
|
+
body: dict | None = None, **params: Any) -> dict:
|
|
625
|
+
"""Delete documents matching a filter query."""
|
|
626
|
+
ns = index or self._namespace
|
|
627
|
+
query = body.get("query", {}) if body else {}
|
|
628
|
+
filter_clause = self._extract_filter(body) or query
|
|
629
|
+
kosha_body = {"namespace": ns, "filter": filter_clause}
|
|
630
|
+
result = self._request("POST", "delete", body=kosha_body)
|
|
631
|
+
deleted = result.get("deleted", 0)
|
|
632
|
+
return {
|
|
633
|
+
"deleted": deleted,
|
|
634
|
+
"total": deleted,
|
|
635
|
+
"failures": [],
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
# ── Update by query ────────────────────────────────────────────────────
|
|
639
|
+
|
|
640
|
+
def update_by_query(self, index: str | None = None,
|
|
641
|
+
body: dict | None = None, **params: Any) -> dict:
|
|
642
|
+
"""Update documents matching a query.
|
|
643
|
+
|
|
644
|
+
Supports simple ``ctx._source.X = ctx._source.Y`` (field copy)
|
|
645
|
+
and ``ctx._source.X = 'value'`` (literal set) scripts.
|
|
646
|
+
"""
|
|
647
|
+
ns = index or self._namespace
|
|
648
|
+
query = (body or {}).get("query", {})
|
|
649
|
+
script = (body or {}).get("script", {})
|
|
650
|
+
source = script.get("source", "").strip()
|
|
651
|
+
|
|
652
|
+
# Parse the script to extract target field and source expression.
|
|
653
|
+
target_field = None
|
|
654
|
+
source_expr = None
|
|
655
|
+
import re
|
|
656
|
+
m = re.match(r"ctx\._source\.(\w+)\s*=\s*(.+)", source)
|
|
657
|
+
if m:
|
|
658
|
+
target_field = m.group(1)
|
|
659
|
+
source_expr = m.group(2).strip().strip("'").strip('"')
|
|
660
|
+
|
|
661
|
+
if not target_field:
|
|
662
|
+
raise NotImplementedError(
|
|
663
|
+
f"Kosha does not support script: {source!r}. "
|
|
664
|
+
"Only simple 'ctx._source.X = ...' patterns work."
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
# Search for matching docs in pages.
|
|
668
|
+
page_size = 100
|
|
669
|
+
from_ = 0
|
|
670
|
+
total_updated = 0
|
|
671
|
+
while True:
|
|
672
|
+
search_body = {
|
|
673
|
+
"query": query,
|
|
674
|
+
"size": page_size,
|
|
675
|
+
"from": from_,
|
|
676
|
+
"_source": True,
|
|
677
|
+
}
|
|
678
|
+
try:
|
|
679
|
+
result = self.search(index=ns, body=search_body)
|
|
680
|
+
except KoshaRequestError:
|
|
681
|
+
break
|
|
682
|
+
|
|
683
|
+
hits = result.get("hits", {}).get("hits", [])
|
|
684
|
+
if not hits:
|
|
685
|
+
break
|
|
686
|
+
|
|
687
|
+
for hit in hits:
|
|
688
|
+
doc_id = hit["_id"]
|
|
689
|
+
source_fields = hit.get("_source", {})
|
|
690
|
+
|
|
691
|
+
# Evaluate the source expression.
|
|
692
|
+
if source_expr in source_fields:
|
|
693
|
+
new_value = source_fields[source_expr]
|
|
694
|
+
else:
|
|
695
|
+
new_value = source_expr
|
|
696
|
+
|
|
697
|
+
# Re-index with updated field.
|
|
698
|
+
source_fields[target_field] = new_value
|
|
699
|
+
self.index(index=ns, id=doc_id, body=source_fields)
|
|
700
|
+
total_updated += 1
|
|
701
|
+
|
|
702
|
+
from_ += page_size
|
|
703
|
+
|
|
704
|
+
# Flush to persist updated segments.
|
|
705
|
+
self._request("POST", "flush", {"namespace": ns})
|
|
706
|
+
|
|
707
|
+
return {
|
|
708
|
+
"updated": total_updated,
|
|
709
|
+
"total": total_updated,
|
|
710
|
+
"failures": [],
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
# ── Scan / Scroll ──────────────────────────────────────────────────────
|
|
714
|
+
|
|
715
|
+
def scroll(self, scroll_id: str = None, scroll: str = "5m", **params):
|
|
716
|
+
"""Compatibility stub — Kosha uses pagination, not scroll cursors."""
|
|
717
|
+
raise NotImplementedError(
|
|
718
|
+
"Kosha does not support scroll cursors. "
|
|
719
|
+
"Use paginated search (from/size) instead."
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
def clear_scroll(self, scroll_id: str, **params):
|
|
723
|
+
"""No-op — Kosha has no scroll state to clear."""
|
|
724
|
+
|
|
725
|
+
# ── kNN no-op (Phase 2) ────────────────────────────────────────────────
|
|
726
|
+
|
|
727
|
+
# ── Index exists / create ──────────────────────────────────────────────
|
|
728
|
+
|
|
729
|
+
def indices(self) -> "IndexOps":
|
|
730
|
+
"""Return an object that mimics ``opensearchpy.client.IndicesClient``."""
|
|
731
|
+
return IndexOps(self)
|
|
732
|
+
|
|
733
|
+
def ping(self, **params: Any) -> bool:
|
|
734
|
+
"""Check if Kosha is reachable."""
|
|
735
|
+
try:
|
|
736
|
+
self._request("GET", "healthz")
|
|
737
|
+
return True
|
|
738
|
+
except Exception:
|
|
739
|
+
return False
|
|
740
|
+
|
|
741
|
+
def close(self) -> None:
|
|
742
|
+
pass
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
# ─── Indices operations (stub) ──────────────────────────────────────────────
|
|
746
|
+
|
|
747
|
+
class IndexOps:
|
|
748
|
+
"""Mimics ``opensearchpy.client.IndicesClient`` for compatibility.
|
|
749
|
+
|
|
750
|
+
Phase 1 is schema-less — Kosha creates namespaces on first write, so
|
|
751
|
+
``create()`` and ``exists()`` are mostly no-ops.
|
|
752
|
+
"""
|
|
753
|
+
|
|
754
|
+
def __init__(self, client: KoshaClient) -> None:
|
|
755
|
+
self._client = client
|
|
756
|
+
|
|
757
|
+
def create(self, index: str, body: dict | None = None, **params: Any) -> dict:
|
|
758
|
+
logger.info("IndexOps.create(%s) — no-op (Kosha is schema-less)", index)
|
|
759
|
+
return {"acknowledged": True, "shards_acknowledged": True, "index": index}
|
|
760
|
+
|
|
761
|
+
def exists(self, index: str, **params: Any) -> bool:
|
|
762
|
+
try:
|
|
763
|
+
self._client.count(index=index)
|
|
764
|
+
return True
|
|
765
|
+
except KoshaRequestError:
|
|
766
|
+
return False
|
|
767
|
+
|
|
768
|
+
def delete(self, index: str, **params: Any) -> dict:
|
|
769
|
+
logger.info("IndexOps.delete(%s) — no-op (Kosha does not support index deletion yet)", index)
|
|
770
|
+
return {"acknowledged": True}
|
|
771
|
+
|
|
772
|
+
def put_mapping(self, index: str, body: dict, **params: Any) -> dict:
|
|
773
|
+
logger.info("IndexOps.put_mapping — no-op (Kosha is schema-less)")
|
|
774
|
+
return {"acknowledged": True}
|
|
775
|
+
|
|
776
|
+
def put_settings(self, index: str, body: dict, **params: Any) -> dict:
|
|
777
|
+
logger.info("IndexOps.put_settings — no-op")
|
|
778
|
+
return {"acknowledged": True}
|
|
779
|
+
|
|
780
|
+
def refresh(self, index: str | None = None, **params: Any) -> dict:
|
|
781
|
+
logger.info("IndexOps.refresh — no-op (Kosha has no refresh cycle)")
|
|
782
|
+
return {"_shards": {"total": 1, "successful": 1, "failed": 0}}
|
|
783
|
+
|
|
784
|
+
def get(self, index: str, **params: Any) -> dict:
|
|
785
|
+
return {index: {"aliases": {}, "mappings": {}, "settings": {}}}
|
|
786
|
+
|
|
787
|
+
def get_mapping(self, index: str | None = None, **params: Any) -> dict:
|
|
788
|
+
return {}
|
|
789
|
+
|
|
790
|
+
def get_settings(self, index: str | None = None, **params: Any) -> dict:
|
|
791
|
+
return {}
|
|
792
|
+
|
|
793
|
+
def flush(self, index: str | None = None, **params: Any) -> dict:
|
|
794
|
+
return {"_shards": {"total": 1, "successful": 1, "failed": 0}}
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
# ─── Error type ─────────────────────────────────────────────────────────────
|
|
798
|
+
|
|
799
|
+
class KoshaRequestError(Exception):
|
|
800
|
+
"""Raised when a Kosha HTTP request fails."""
|
|
801
|
+
|
|
802
|
+
def __init__(self, status_code: int, error: str, info: dict | None = None):
|
|
803
|
+
self.status_code = status_code
|
|
804
|
+
self.error = error
|
|
805
|
+
self.info = info or {}
|
|
806
|
+
super().__init__(f"KoshaRequestError({status_code}): {error}")
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kosha-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: OpenSearch-compatible Python client for the Kosha search engine
|
|
5
|
+
Author-email: DecoverAI <engineering@decoverai.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/ravi-decoverai/Kosha
|
|
8
|
+
Project-URL: Source, https://github.com/ravi-decoverai/Kosha
|
|
9
|
+
Requires-Python: >=3.10
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "kosha-client"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "OpenSearch-compatible Python client for the Kosha search engine"
|
|
5
|
+
requires-python = ">=3.10"
|
|
6
|
+
license = { text = "Apache-2.0" }
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "DecoverAI", email = "engineering@decoverai.com" },
|
|
9
|
+
]
|
|
10
|
+
urls = { Homepage = "https://github.com/ravi-decoverai/Kosha", Source = "https://github.com/ravi-decoverai/Kosha" }
|
|
11
|
+
dependencies = []
|
|
12
|
+
|
|
13
|
+
[build-system]
|
|
14
|
+
requires = ["setuptools"]
|
|
15
|
+
build-backend = "setuptools.build_meta"
|
|
16
|
+
|
|
17
|
+
[tool.setuptools.packages.find]
|
|
18
|
+
where = ["."]
|