svo-client-dev 4.0.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- svo_client/__init__.py +75 -0
- svo_client/_version.py +3 -0
- svo_client/aiohttp_ws_max_message.py +50 -0
- svo_client/chunk_presets.py +36 -0
- svo_client/cli.py +225 -0
- svo_client/cli_chunk.py +386 -0
- svo_client/client.py +1104 -0
- svo_client/config.py +149 -0
- svo_client/connect_defaults.py +230 -0
- svo_client/constants.py +4 -0
- svo_client/core.py +330 -0
- svo_client/debug_ndjson.py +14 -0
- svo_client/exceptions.py +48 -0
- svo_client/file_processor.py +58 -0
- svo_client/transfer_precheck.py +91 -0
- svo_client_dev-4.0.21.dist-info/METADATA +363 -0
- svo_client_dev-4.0.21.dist-info/RECORD +20 -0
- svo_client_dev-4.0.21.dist-info/WHEEL +5 -0
- svo_client_dev-4.0.21.dist-info/entry_points.txt +3 -0
- svo_client_dev-4.0.21.dist-info/top_level.txt +1 -0
svo_client/client.py
ADDED
|
@@ -0,0 +1,1104 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Asynchronous SVO Chunker client. All network I/O goes only through
|
|
3
|
+
``mcp_proxy_adapter.client.jsonrpc_client.JsonRpcClient`` (no ad-hoc HTTP
|
|
4
|
+
clients in this package).
|
|
5
|
+
|
|
6
|
+
Commands use ``execute_command_unified(..., auto_poll=True)``: the adapter
|
|
7
|
+
delivers command-session completion on WebSocket ``/ws``. JSON-RPC submission
|
|
8
|
+
and binary artifact fetch (e.g. outbound JSON after ``file_chunk``) use the
|
|
9
|
+
adapter's HTTP(S) transport.
|
|
10
|
+
|
|
11
|
+
This package does **not** call the embedding / vectorization HTTP API
|
|
12
|
+
(``embed_client``) directly; vectorization runs inside the chunker service,
|
|
13
|
+
which uses embed-client's in-process execute_async/WebSocket completion path.
|
|
14
|
+
With mcp-proxy-adapter **>= 8.10**, queued commands still return a ``job_id``
|
|
15
|
+
and the client receives their full results after WebSocket completion.
|
|
16
|
+
|
|
17
|
+
chunk(texts) calls server method "chunk" (list of texts in, list of results out).
|
|
18
|
+
``chunk`` runs in the server process; the long-running ``file_chunk`` command
|
|
19
|
+
uses an isolated queue worker. Server command sessions expose completion via /ws.
|
|
20
|
+
|
|
21
|
+
Author: Vasiliy Zdanovskiy
|
|
22
|
+
email: vasilyvz@gmail.com
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import asyncio
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import tempfile
|
|
29
|
+
import copy
|
|
30
|
+
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional
|
|
31
|
+
|
|
32
|
+
from chunk_metadata_adapter.semantic_chunk import SemanticChunk
|
|
33
|
+
|
|
34
|
+
from svo_client.chunk_presets import warn_if_unknown_chunk_set
|
|
35
|
+
from svo_client.core import ClientCore
|
|
36
|
+
from svo_client.exceptions import (
|
|
37
|
+
SvoChunkingError,
|
|
38
|
+
SvoClientError,
|
|
39
|
+
SvoConnectionError,
|
|
40
|
+
SvoTimeoutError,
|
|
41
|
+
SvoValidationError,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
if TYPE_CHECKING:
|
|
45
|
+
from mcp_proxy_adapter.client.jsonrpc_client.ws_job_status import (
|
|
46
|
+
BidirectionalWsChannel,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _validate_embedding_pair(model: Optional[str], dimension: Optional[int]) -> None:
|
|
51
|
+
"""Validate the optional model/dimension request pair before any RPC."""
|
|
52
|
+
if (model is None) != (dimension is None):
|
|
53
|
+
raise SvoValidationError(
|
|
54
|
+
"Parameters 'model' and 'dimension' must be provided together "
|
|
55
|
+
"or both omitted"
|
|
56
|
+
)
|
|
57
|
+
if model is not None and (not isinstance(model, str) or not model.strip()):
|
|
58
|
+
raise SvoValidationError(
|
|
59
|
+
"Parameter 'model' must be a non-empty string when provided"
|
|
60
|
+
)
|
|
61
|
+
if dimension is not None and (
|
|
62
|
+
isinstance(dimension, bool) or not isinstance(dimension, int) or dimension <= 0
|
|
63
|
+
):
|
|
64
|
+
raise SvoValidationError(
|
|
65
|
+
"Parameter 'dimension' must be a positive integer when provided"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _unwrap_queued_command_result(raw: Dict[str, Any]) -> Dict[str, Any]:
|
|
70
|
+
"""If the adapter returned a job envelope, return the inner command payload."""
|
|
71
|
+
nested = raw.get("result")
|
|
72
|
+
if isinstance(nested, dict) and (
|
|
73
|
+
"success" in nested
|
|
74
|
+
or nested.get("error") is not None
|
|
75
|
+
or "results" in nested
|
|
76
|
+
or "body" in nested
|
|
77
|
+
or nested.get("transfer_id")
|
|
78
|
+
or isinstance(nested.get("data"), dict)
|
|
79
|
+
):
|
|
80
|
+
return nested
|
|
81
|
+
return raw
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def unwrap_command_result(raw: Dict[str, Any]) -> Dict[str, Any]:
|
|
85
|
+
"""Unwrap transport, queue and SuccessResult envelopes without losing errors."""
|
|
86
|
+
current = raw
|
|
87
|
+
for _ in range(12):
|
|
88
|
+
if not isinstance(current, dict):
|
|
89
|
+
raise SvoChunkingError("Invalid command response envelope")
|
|
90
|
+
if (
|
|
91
|
+
current.get("error") is not None
|
|
92
|
+
or current.get("mime")
|
|
93
|
+
or current.get("type") in ("file", "file_chunk")
|
|
94
|
+
or isinstance(current.get("results"), list)
|
|
95
|
+
):
|
|
96
|
+
return current
|
|
97
|
+
nested = current.get("result")
|
|
98
|
+
if isinstance(nested, dict):
|
|
99
|
+
current = nested
|
|
100
|
+
continue
|
|
101
|
+
nested = current.get("data")
|
|
102
|
+
if isinstance(nested, dict) and ("success" in current or len(current) == 1):
|
|
103
|
+
current = nested
|
|
104
|
+
continue
|
|
105
|
+
return current
|
|
106
|
+
raise SvoChunkingError("Command response nesting is too deep")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _semantic_chunk_dict_from_wire(raw: Dict[str, Any]) -> Dict[str, Any]:
|
|
110
|
+
"""
|
|
111
|
+
Validate and normalize a chunk dict from JSON-RPC / NDJSON using
|
|
112
|
+
``chunk_metadata_adapter`` (same contract as the server).
|
|
113
|
+
"""
|
|
114
|
+
semantic_chunk, errors = SemanticChunk.validate_and_fill(raw)
|
|
115
|
+
if errors:
|
|
116
|
+
msg = errors.get("error") if isinstance(errors, dict) else str(errors)
|
|
117
|
+
raise SvoChunkingError(
|
|
118
|
+
f"chunk: SemanticChunk validation failed: {msg}"
|
|
119
|
+
) from None
|
|
120
|
+
dumped = semantic_chunk.model_dump()
|
|
121
|
+
for policy_key in (
|
|
122
|
+
"chunk_set",
|
|
123
|
+
"chunk_policy_version",
|
|
124
|
+
"atom_count",
|
|
125
|
+
"merged_atom_kinds",
|
|
126
|
+
"ignored_structural_markers",
|
|
127
|
+
"global_start",
|
|
128
|
+
"global_end",
|
|
129
|
+
):
|
|
130
|
+
if raw.get(policy_key) is not None:
|
|
131
|
+
dumped[policy_key] = raw[policy_key]
|
|
132
|
+
return dumped
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _normalize_chunk_dict_list(chunks: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
136
|
+
return [_semantic_chunk_dict_from_wire(c) for c in chunks]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _extract_timing_breakdown(result: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
140
|
+
result_data = result.get("result")
|
|
141
|
+
if not isinstance(result_data, dict):
|
|
142
|
+
return None
|
|
143
|
+
tb = result_data.get("timing_breakdown")
|
|
144
|
+
if isinstance(tb, dict):
|
|
145
|
+
return tb
|
|
146
|
+
data = result_data.get("data", {})
|
|
147
|
+
if isinstance(data, dict):
|
|
148
|
+
tb = data.get("timing_breakdown")
|
|
149
|
+
if isinstance(tb, dict):
|
|
150
|
+
return tb
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _extract_chunks_from_result(result: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
155
|
+
if "result" not in result:
|
|
156
|
+
if "error" in result:
|
|
157
|
+
raise SvoChunkingError(
|
|
158
|
+
result.get("message", result.get("error", "Unknown error"))
|
|
159
|
+
)
|
|
160
|
+
raise SvoChunkingError("No chunks returned from server")
|
|
161
|
+
|
|
162
|
+
result_data = result["result"]
|
|
163
|
+
|
|
164
|
+
if isinstance(result_data, list) and result_data:
|
|
165
|
+
return result_data
|
|
166
|
+
|
|
167
|
+
if isinstance(result_data, dict):
|
|
168
|
+
nested = result_data.get("result", {})
|
|
169
|
+
if isinstance(nested, dict):
|
|
170
|
+
data = nested.get("data", {})
|
|
171
|
+
if isinstance(data, dict):
|
|
172
|
+
chunks = data.get("chunks", [])
|
|
173
|
+
if chunks and isinstance(chunks, list):
|
|
174
|
+
return chunks
|
|
175
|
+
chunks = result_data.get("chunks", [])
|
|
176
|
+
if chunks and isinstance(chunks, list):
|
|
177
|
+
return chunks
|
|
178
|
+
data = result_data.get("data", {})
|
|
179
|
+
if isinstance(data, dict):
|
|
180
|
+
chunks = data.get("chunks", [])
|
|
181
|
+
if chunks and isinstance(chunks, list):
|
|
182
|
+
return chunks
|
|
183
|
+
if "error" in result_data:
|
|
184
|
+
raise SvoChunkingError(
|
|
185
|
+
result_data.get("message", result_data.get("error", "Chunking failed"))
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
raise SvoChunkingError("No chunks in server response")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _ndjson_body_to_per_text_chunk_lists(
|
|
192
|
+
body: str,
|
|
193
|
+
text_chunk_counts: List[int],
|
|
194
|
+
) -> List[List[Dict[str, Any]]]:
|
|
195
|
+
"""Split flat NDJSON chunk lines using per-text ``text_chunk_counts``."""
|
|
196
|
+
lines = [ln for ln in body.split("\n") if ln.strip()]
|
|
197
|
+
chunks_flat: List[Dict[str, Any]] = []
|
|
198
|
+
for ln in lines:
|
|
199
|
+
try:
|
|
200
|
+
parsed = json.loads(ln)
|
|
201
|
+
except json.JSONDecodeError:
|
|
202
|
+
raise SvoChunkingError("chunk: invalid NDJSON line in body") from None
|
|
203
|
+
if not isinstance(parsed, dict):
|
|
204
|
+
raise SvoChunkingError("chunk: NDJSON line must be a JSON object")
|
|
205
|
+
chunks_flat.append(_semantic_chunk_dict_from_wire(parsed))
|
|
206
|
+
out: List[List[Dict[str, Any]]] = []
|
|
207
|
+
pos = 0
|
|
208
|
+
for cnt in text_chunk_counts:
|
|
209
|
+
if isinstance(cnt, bool) or not isinstance(cnt, int) or cnt < 0:
|
|
210
|
+
raise SvoChunkingError("chunk: invalid text_chunk_counts")
|
|
211
|
+
out.append(chunks_flat[pos : pos + cnt])
|
|
212
|
+
pos += cnt
|
|
213
|
+
if pos != len(chunks_flat):
|
|
214
|
+
raise SvoChunkingError(
|
|
215
|
+
"chunk: text_chunk_counts does not match NDJSON body line count"
|
|
216
|
+
)
|
|
217
|
+
return out
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def parse_chunk_ndjson(
|
|
221
|
+
body: str, text_chunk_counts: List[int]
|
|
222
|
+
) -> List[List[Dict[str, Any]]]:
|
|
223
|
+
"""Parse exact chunk metadata and split NDJSON by original input index."""
|
|
224
|
+
if not isinstance(body, str) or not isinstance(text_chunk_counts, list):
|
|
225
|
+
raise SvoChunkingError("chunk: invalid NDJSON body or text_chunk_counts")
|
|
226
|
+
return _ndjson_body_to_per_text_chunk_lists(body, text_chunk_counts)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def parse_chunk_ndjson_file(
|
|
230
|
+
path: str, text_chunk_counts: Optional[List[int]] = None
|
|
231
|
+
) -> List[List[Dict[str, Any]]]:
|
|
232
|
+
"""Parse an artifact; absent counts return its chunks as one sequence."""
|
|
233
|
+
with open(path, encoding="utf-8", newline="") as stream:
|
|
234
|
+
body = stream.read()
|
|
235
|
+
if text_chunk_counts is None:
|
|
236
|
+
text_chunk_counts = [sum(bool(line.strip()) for line in body.split("\n"))]
|
|
237
|
+
return parse_chunk_ndjson(body, text_chunk_counts)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _ndjson_path_to_per_text_chunk_lists(
|
|
241
|
+
path: str,
|
|
242
|
+
text_chunk_counts: List[int],
|
|
243
|
+
) -> List[List[Dict[str, Any]]]:
|
|
244
|
+
"""Split flat NDJSON file using per-text ``text_chunk_counts`` (line-oriented read)."""
|
|
245
|
+
chunks_flat: List[Dict[str, Any]] = []
|
|
246
|
+
with open(path, encoding="utf-8") as f:
|
|
247
|
+
for ln in f:
|
|
248
|
+
ln = ln.strip()
|
|
249
|
+
if not ln:
|
|
250
|
+
continue
|
|
251
|
+
try:
|
|
252
|
+
parsed = json.loads(ln)
|
|
253
|
+
except json.JSONDecodeError:
|
|
254
|
+
raise SvoChunkingError("chunk: invalid NDJSON line in body") from None
|
|
255
|
+
if not isinstance(parsed, dict):
|
|
256
|
+
raise SvoChunkingError("chunk: NDJSON line must be a JSON object")
|
|
257
|
+
chunks_flat.append(_semantic_chunk_dict_from_wire(parsed))
|
|
258
|
+
out: List[List[Dict[str, Any]]] = []
|
|
259
|
+
pos = 0
|
|
260
|
+
for cnt in text_chunk_counts:
|
|
261
|
+
if cnt < 0:
|
|
262
|
+
raise SvoChunkingError("chunk: invalid text_chunk_counts")
|
|
263
|
+
out.append(chunks_flat[pos : pos + cnt])
|
|
264
|
+
pos += cnt
|
|
265
|
+
if pos != len(chunks_flat):
|
|
266
|
+
raise SvoChunkingError(
|
|
267
|
+
"chunk: text_chunk_counts does not match NDJSON body line count"
|
|
268
|
+
)
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
async def _chunk_success_ndjson_per_text_lists(
|
|
273
|
+
core: ClientCore,
|
|
274
|
+
raw: Dict[str, Any],
|
|
275
|
+
) -> List[List[Dict[str, Any]]]:
|
|
276
|
+
"""
|
|
277
|
+
Resolve NDJSON payload from inline ``body`` or ``transfer_id`` download.
|
|
278
|
+
"""
|
|
279
|
+
counts_raw = raw.get("text_chunk_counts")
|
|
280
|
+
if not isinstance(counts_raw, list):
|
|
281
|
+
raise SvoChunkingError("chunk: invalid text_chunk_counts")
|
|
282
|
+
if any(isinstance(x, bool) or not isinstance(x, int) or x < 0 for x in counts_raw):
|
|
283
|
+
raise SvoChunkingError("chunk: invalid text_chunk_counts")
|
|
284
|
+
counts = counts_raw
|
|
285
|
+
tid = raw.get("transfer_id")
|
|
286
|
+
if isinstance(tid, str) and tid.strip():
|
|
287
|
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".jsonl")
|
|
288
|
+
tmp_path = tmp.name
|
|
289
|
+
tmp.close()
|
|
290
|
+
try:
|
|
291
|
+
await core.download_transfer_to_path(tid.strip(), tmp_path)
|
|
292
|
+
return _ndjson_path_to_per_text_chunk_lists(tmp_path, counts)
|
|
293
|
+
finally:
|
|
294
|
+
try:
|
|
295
|
+
os.unlink(tmp_path)
|
|
296
|
+
except OSError:
|
|
297
|
+
pass
|
|
298
|
+
body = raw.get("body")
|
|
299
|
+
if isinstance(body, str):
|
|
300
|
+
return _ndjson_body_to_per_text_chunk_lists(body, counts)
|
|
301
|
+
raise SvoChunkingError("chunk: missing NDJSON body and transfer_id")
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _extract_batch_results(raw: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
305
|
+
"""
|
|
306
|
+
Extract batch item results from top-level ``results`` or nested ``data.results``.
|
|
307
|
+
|
|
308
|
+
Canonical:
|
|
309
|
+
{"result": {"success": true, "results": [...]}}
|
|
310
|
+
Alternate nested shape:
|
|
311
|
+
{"result": {"success": true, "data": {"success": true, "results": [...]}}}
|
|
312
|
+
"""
|
|
313
|
+
results_list = raw.get("results")
|
|
314
|
+
if isinstance(results_list, list):
|
|
315
|
+
return results_list
|
|
316
|
+
|
|
317
|
+
nested_result = raw.get("result")
|
|
318
|
+
if isinstance(nested_result, list):
|
|
319
|
+
return nested_result
|
|
320
|
+
|
|
321
|
+
nested_data = raw.get("data")
|
|
322
|
+
if isinstance(nested_data, dict):
|
|
323
|
+
data_results = nested_data.get("results")
|
|
324
|
+
if isinstance(data_results, list):
|
|
325
|
+
return data_results
|
|
326
|
+
|
|
327
|
+
return []
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def build_file_chunk_params(
|
|
331
|
+
*,
|
|
332
|
+
filepath: Optional[str] = None,
|
|
333
|
+
filename: Optional[str] = None,
|
|
334
|
+
file_content_base64: Optional[str] = None,
|
|
335
|
+
transfer_id: Optional[str] = None,
|
|
336
|
+
filter_name: Optional[str] = None,
|
|
337
|
+
window: int = 3,
|
|
338
|
+
chunk_type: Optional[str] = None,
|
|
339
|
+
language: Optional[str] = None,
|
|
340
|
+
source_id: Optional[str] = None,
|
|
341
|
+
project: Optional[str] = None,
|
|
342
|
+
unit_id: Optional[str] = None,
|
|
343
|
+
chunking_version: str = "1.0",
|
|
344
|
+
use_sv: bool = False,
|
|
345
|
+
aggregation_mode: Optional[str] = None,
|
|
346
|
+
respect_structural_blocks: Optional[bool] = None,
|
|
347
|
+
chunk_set: Optional[str] = None,
|
|
348
|
+
model: Optional[str] = None,
|
|
349
|
+
dimension: Optional[int] = None,
|
|
350
|
+
chunk_only: Optional[bool] = None,
|
|
351
|
+
split_level: Optional[str] = None,
|
|
352
|
+
max_chars: Optional[int] = None,
|
|
353
|
+
) -> Dict[str, Any]:
|
|
354
|
+
"""
|
|
355
|
+
Build JSON-RPC parameters for the ``file_chunk`` command.
|
|
356
|
+
|
|
357
|
+
File acquisition matches ``file`` (oneOf): ``filepath``, or ``filename`` +
|
|
358
|
+
``file_content_base64``, or ``transfer_id`` + ``filename``. Chunk fields match
|
|
359
|
+
``chunk_batch`` without ``texts``.
|
|
360
|
+
"""
|
|
361
|
+
_validate_embedding_pair(model, dimension)
|
|
362
|
+
|
|
363
|
+
params: Dict[str, Any] = {
|
|
364
|
+
"window": window,
|
|
365
|
+
"use_sv": use_sv,
|
|
366
|
+
}
|
|
367
|
+
if filepath is not None:
|
|
368
|
+
params["filepath"] = filepath
|
|
369
|
+
if filename is not None:
|
|
370
|
+
params["filename"] = filename
|
|
371
|
+
if file_content_base64 is not None:
|
|
372
|
+
params["file_content_base64"] = file_content_base64
|
|
373
|
+
if transfer_id is not None:
|
|
374
|
+
params["transfer_id"] = transfer_id
|
|
375
|
+
if filter_name is not None:
|
|
376
|
+
params["filter_name"] = filter_name
|
|
377
|
+
if chunk_type is not None:
|
|
378
|
+
params["type"] = chunk_type
|
|
379
|
+
if language is not None:
|
|
380
|
+
params["language"] = language
|
|
381
|
+
if source_id is not None:
|
|
382
|
+
params["source_id"] = source_id
|
|
383
|
+
if project is not None:
|
|
384
|
+
params["project"] = project
|
|
385
|
+
if unit_id is not None:
|
|
386
|
+
params["unit_id"] = unit_id
|
|
387
|
+
if chunking_version is not None:
|
|
388
|
+
params["chunking_version"] = chunking_version
|
|
389
|
+
if aggregation_mode is not None:
|
|
390
|
+
params["aggregation_mode"] = aggregation_mode
|
|
391
|
+
if respect_structural_blocks is not None:
|
|
392
|
+
params["respect_structural_blocks"] = respect_structural_blocks
|
|
393
|
+
warn_if_unknown_chunk_set(chunk_set)
|
|
394
|
+
if chunk_set is not None:
|
|
395
|
+
params["chunk_set"] = chunk_set
|
|
396
|
+
if model is not None:
|
|
397
|
+
params["model"] = model
|
|
398
|
+
params["dimension"] = dimension
|
|
399
|
+
for name, value in (
|
|
400
|
+
("chunk_only", chunk_only),
|
|
401
|
+
("split_level", split_level),
|
|
402
|
+
("max_chars", max_chars),
|
|
403
|
+
):
|
|
404
|
+
if value is not None:
|
|
405
|
+
params[name] = value
|
|
406
|
+
return params
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _unwrap_file_chunk_payload_dict(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
410
|
+
"""
|
|
411
|
+
Walk nested ``{"success", "data": {...}}`` envelopes until the ``file_chunk``
|
|
412
|
+
contract dict (``type == "file_chunk"``) is at the current level.
|
|
413
|
+
|
|
414
|
+
WebSocket ``job_completed`` results are normalized in
|
|
415
|
+
``mcp_proxy_adapter`` ``execute_command_unified`` by stripping a single
|
|
416
|
+
``data`` wrapper from the ``result`` field. Long-running jobs may still
|
|
417
|
+
carry one extra ``SuccessResult``-shaped layer, so the client must peel
|
|
418
|
+
repeatedly (same idea as :func:`_extract_chunks_from_result` for ``chunk``).
|
|
419
|
+
"""
|
|
420
|
+
cur: Any = payload
|
|
421
|
+
for _ in range(8):
|
|
422
|
+
if not isinstance(cur, dict):
|
|
423
|
+
break
|
|
424
|
+
if cur.get("type") == "file_chunk":
|
|
425
|
+
return cur
|
|
426
|
+
nxt = cur.get("data")
|
|
427
|
+
if isinstance(nxt, dict):
|
|
428
|
+
cur = nxt
|
|
429
|
+
continue
|
|
430
|
+
break
|
|
431
|
+
if not isinstance(cur, dict):
|
|
432
|
+
raise SvoChunkingError("file_chunk: invalid result payload")
|
|
433
|
+
return cur
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def extract_file_chunk_success_payload(unified: Dict[str, Any]) -> Dict[str, Any]:
|
|
437
|
+
"""
|
|
438
|
+
Extract the successful ``file_chunk`` data dict from ``ClientCore.call`` /
|
|
439
|
+
``execute_command_unified`` return value.
|
|
440
|
+
"""
|
|
441
|
+
inner = unified.get("result")
|
|
442
|
+
if not isinstance(inner, dict):
|
|
443
|
+
raise SvoChunkingError("file_chunk: missing or invalid result")
|
|
444
|
+
|
|
445
|
+
# Queued jobs: top-level ``result`` is { job_id, command, result: {...}, status }
|
|
446
|
+
nested_cmd = inner.get("result")
|
|
447
|
+
if isinstance(nested_cmd, dict) and (
|
|
448
|
+
"success" in nested_cmd
|
|
449
|
+
or nested_cmd.get("error") is not None
|
|
450
|
+
or isinstance(nested_cmd.get("data"), dict)
|
|
451
|
+
):
|
|
452
|
+
inner = nested_cmd
|
|
453
|
+
|
|
454
|
+
if inner.get("error"):
|
|
455
|
+
err = inner["error"]
|
|
456
|
+
if isinstance(err, dict):
|
|
457
|
+
msg = err.get("message", str(err))
|
|
458
|
+
else:
|
|
459
|
+
msg = str(err)
|
|
460
|
+
raise SvoChunkingError(msg, error=err)
|
|
461
|
+
|
|
462
|
+
if isinstance(inner.get("data"), dict):
|
|
463
|
+
payload = inner["data"]
|
|
464
|
+
else:
|
|
465
|
+
payload = inner
|
|
466
|
+
|
|
467
|
+
payload = _unwrap_file_chunk_payload_dict(payload)
|
|
468
|
+
|
|
469
|
+
if payload.get("status") == "error" and payload.get("message") is not None:
|
|
470
|
+
raise SvoChunkingError(str(payload["message"]))
|
|
471
|
+
|
|
472
|
+
if payload.get("type") != "file_chunk":
|
|
473
|
+
raise SvoChunkingError(
|
|
474
|
+
f"file_chunk: result type mismatch (got {payload.get('type')!r})"
|
|
475
|
+
)
|
|
476
|
+
if not payload.get("success", False):
|
|
477
|
+
raise SvoChunkingError("file_chunk: server reported failure")
|
|
478
|
+
tid = payload.get("transfer_id")
|
|
479
|
+
if not isinstance(tid, str) or not tid:
|
|
480
|
+
raise SvoChunkingError("file_chunk: missing transfer_id")
|
|
481
|
+
|
|
482
|
+
return payload
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
class SvoChunkerClient:
|
|
486
|
+
"""
|
|
487
|
+
Asynchronous client for SVO Chunker.
|
|
488
|
+
|
|
489
|
+
File vectorization uses ``file_chunk`` (server RPC). ``chunk`` runs in the
|
|
490
|
+
server process and ``file_chunk`` uses a queue worker; :class:`ClientCore` /
|
|
491
|
+
``JsonRpcClient`` command sessions deliver completion over WebSocket. Text
|
|
492
|
+
embedding inside the pipeline is handled by the server (embed-client); this
|
|
493
|
+
client only drives chunker RPC.
|
|
494
|
+
Optional artifact download after ``file_chunk`` uses
|
|
495
|
+
:meth:`ClientCore.download_transfer_to_path` (declared-size precheck,
|
|
496
|
+
then ``JsonRpcClient.download_file``). Use ``open_ws_channel()`` for a
|
|
497
|
+
long-lived WebSocket when needed.
|
|
498
|
+
"""
|
|
499
|
+
|
|
500
|
+
def __init__(
|
|
501
|
+
self,
|
|
502
|
+
protocol: str = "https",
|
|
503
|
+
host: str = "127.0.0.1",
|
|
504
|
+
port: int = 8009,
|
|
505
|
+
token_header: Optional[str] = None,
|
|
506
|
+
token: Optional[str] = None,
|
|
507
|
+
cert: Optional[str] = None,
|
|
508
|
+
key: Optional[str] = None,
|
|
509
|
+
ca: Optional[str] = None,
|
|
510
|
+
check_hostname: bool = False,
|
|
511
|
+
timeout: Optional[float] = 300.0,
|
|
512
|
+
poll_interval: float = 1.0,
|
|
513
|
+
certs_dir: Optional[str] = None,
|
|
514
|
+
file_batch_size: Optional[int] = None,
|
|
515
|
+
):
|
|
516
|
+
self._core = ClientCore(
|
|
517
|
+
protocol=protocol,
|
|
518
|
+
host=host,
|
|
519
|
+
port=port,
|
|
520
|
+
token_header=token_header,
|
|
521
|
+
token=token,
|
|
522
|
+
cert=cert,
|
|
523
|
+
key=key,
|
|
524
|
+
ca=ca,
|
|
525
|
+
check_hostname=check_hostname,
|
|
526
|
+
timeout=timeout,
|
|
527
|
+
certs_dir=certs_dir,
|
|
528
|
+
)
|
|
529
|
+
self.timeout = timeout
|
|
530
|
+
self.poll_interval = poll_interval
|
|
531
|
+
self._last_timing_breakdown: Optional[Dict[str, Any]] = None
|
|
532
|
+
self._file_batch_size = file_batch_size
|
|
533
|
+
|
|
534
|
+
@property
|
|
535
|
+
def file_batch_size(self) -> Optional[int]:
|
|
536
|
+
"""Optional value copied from the client config (JSON/env), for documentation only.
|
|
537
|
+
|
|
538
|
+
The server ``file_chunk`` batch width is controlled by **server** ``config`` key
|
|
539
|
+
``file_batch_size``. This property is **not** sent on RPC; it exists so callers
|
|
540
|
+
can display or log the expected batch size without re-reading the server file.
|
|
541
|
+
"""
|
|
542
|
+
return self._file_batch_size
|
|
543
|
+
|
|
544
|
+
@property
|
|
545
|
+
def last_timing_breakdown(self) -> Optional[Dict[str, Any]]:
|
|
546
|
+
"""Server-side timing breakdown from the last chunk() call, if present."""
|
|
547
|
+
return self._last_timing_breakdown
|
|
548
|
+
|
|
549
|
+
async def call(
|
|
550
|
+
self,
|
|
551
|
+
command: str,
|
|
552
|
+
params: Optional[Dict[str, Any]] = None,
|
|
553
|
+
*,
|
|
554
|
+
status_hook: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
|
555
|
+
) -> Dict[str, Any]:
|
|
556
|
+
try:
|
|
557
|
+
return await self._core.call(command, params or {}, status_hook=status_hook)
|
|
558
|
+
except SvoClientError:
|
|
559
|
+
raise
|
|
560
|
+
except Exception as e:
|
|
561
|
+
raise SvoConnectionError(f"Command '{command}' failed: {e}") from e
|
|
562
|
+
|
|
563
|
+
execute = call
|
|
564
|
+
|
|
565
|
+
async def help(self, command_name: Optional[str] = None) -> Dict[str, Any]:
|
|
566
|
+
return await self._core.get_help(command_name)
|
|
567
|
+
|
|
568
|
+
async def list_commands(self) -> List[str]:
|
|
569
|
+
return await self._core.list_commands()
|
|
570
|
+
|
|
571
|
+
async def command_contract(self, command: str) -> Dict[str, Any]:
|
|
572
|
+
"""Fetch a live command contract including every inherited API parameter."""
|
|
573
|
+
response = await self.help(command)
|
|
574
|
+
current = response
|
|
575
|
+
for _ in range(12):
|
|
576
|
+
if not isinstance(current, dict):
|
|
577
|
+
break
|
|
578
|
+
if isinstance(current.get("schema"), dict):
|
|
579
|
+
return copy.deepcopy(current)
|
|
580
|
+
commands = current.get("commands")
|
|
581
|
+
if isinstance(commands, dict) and isinstance(commands.get(command), dict):
|
|
582
|
+
current = commands[command]
|
|
583
|
+
continue
|
|
584
|
+
nested = current.get("data")
|
|
585
|
+
if not isinstance(nested, dict):
|
|
586
|
+
nested = current.get("result")
|
|
587
|
+
if not isinstance(nested, dict):
|
|
588
|
+
break
|
|
589
|
+
current = nested
|
|
590
|
+
raise SvoValidationError(f"No live schema returned for command {command!r}")
|
|
591
|
+
|
|
592
|
+
async def prepare_command(
|
|
593
|
+
self, command: str, params: Dict[str, Any]
|
|
594
|
+
) -> Dict[str, Any]:
|
|
595
|
+
"""Validate against the live schema without dispatch or default injection.
|
|
596
|
+
|
|
597
|
+
This is the explicit schema-backed surface for inherited adapter commands,
|
|
598
|
+
including nested settings. Omission, null, false and zero are preserved.
|
|
599
|
+
"""
|
|
600
|
+
from jsonschema import Draft7Validator, FormatChecker
|
|
601
|
+
|
|
602
|
+
contract = await self.command_contract(command)
|
|
603
|
+
schema = contract["schema"]
|
|
604
|
+
if not isinstance(params, dict):
|
|
605
|
+
raise SvoValidationError("Command parameters must be an object")
|
|
606
|
+
unknown = set(params) - set(schema.get("properties", {}))
|
|
607
|
+
if unknown:
|
|
608
|
+
raise SvoValidationError(
|
|
609
|
+
f"Unknown parameters for {command}: {sorted(unknown)}"
|
|
610
|
+
)
|
|
611
|
+
errors = list(
|
|
612
|
+
Draft7Validator(schema, format_checker=FormatChecker()).iter_errors(params)
|
|
613
|
+
)
|
|
614
|
+
if errors:
|
|
615
|
+
errors.sort(
|
|
616
|
+
key=lambda error: (error.validator != "required", str(list(error.path)))
|
|
617
|
+
)
|
|
618
|
+
raise SvoValidationError(errors[0].message)
|
|
619
|
+
return copy.deepcopy(params)
|
|
620
|
+
|
|
621
|
+
async def chunk(
|
|
622
|
+
self,
|
|
623
|
+
text: str,
|
|
624
|
+
window: int = 3,
|
|
625
|
+
language: Optional[str] = None,
|
|
626
|
+
source_id: Optional[str] = None,
|
|
627
|
+
project: Optional[str] = None,
|
|
628
|
+
unit_id: Optional[str] = None,
|
|
629
|
+
chunking_version: str = "1.0",
|
|
630
|
+
use_sv: bool = False,
|
|
631
|
+
chunk_type: Optional[str] = None,
|
|
632
|
+
aggregation_mode: Optional[str] = None,
|
|
633
|
+
respect_structural_blocks: Optional[bool] = None,
|
|
634
|
+
chunk_set: Optional[str] = None,
|
|
635
|
+
model: Optional[str] = None,
|
|
636
|
+
dimension: Optional[int] = None,
|
|
637
|
+
chunk_only: Optional[bool] = None,
|
|
638
|
+
split_level: Optional[str] = None,
|
|
639
|
+
max_chars: Optional[int] = None,
|
|
640
|
+
) -> List[Dict[str, Any]]:
|
|
641
|
+
if not isinstance(text, str) or (
|
|
642
|
+
not text.strip() and split_level is None and max_chars is None
|
|
643
|
+
):
|
|
644
|
+
raise SvoValidationError("Text cannot be empty")
|
|
645
|
+
|
|
646
|
+
_validate_embedding_pair(model, dimension)
|
|
647
|
+
|
|
648
|
+
warn_if_unknown_chunk_set(chunk_set)
|
|
649
|
+
|
|
650
|
+
params: Dict[str, Any] = {
|
|
651
|
+
"texts": [text],
|
|
652
|
+
"window": window,
|
|
653
|
+
"use_sv": use_sv,
|
|
654
|
+
}
|
|
655
|
+
if chunk_type is not None:
|
|
656
|
+
params["type"] = chunk_type
|
|
657
|
+
if language is not None:
|
|
658
|
+
params["language"] = language
|
|
659
|
+
if source_id is not None:
|
|
660
|
+
params["source_id"] = source_id
|
|
661
|
+
if project is not None:
|
|
662
|
+
params["project"] = project
|
|
663
|
+
if unit_id is not None:
|
|
664
|
+
params["unit_id"] = unit_id
|
|
665
|
+
if chunking_version is not None:
|
|
666
|
+
params["chunking_version"] = chunking_version
|
|
667
|
+
if aggregation_mode is not None:
|
|
668
|
+
params["aggregation_mode"] = aggregation_mode
|
|
669
|
+
if respect_structural_blocks is not None:
|
|
670
|
+
params["respect_structural_blocks"] = respect_structural_blocks
|
|
671
|
+
if chunk_set is not None:
|
|
672
|
+
params["chunk_set"] = chunk_set
|
|
673
|
+
if model is not None:
|
|
674
|
+
params["model"] = model
|
|
675
|
+
params["dimension"] = dimension
|
|
676
|
+
|
|
677
|
+
for name, value in (
|
|
678
|
+
("chunk_only", chunk_only),
|
|
679
|
+
("split_level", split_level),
|
|
680
|
+
("max_chars", max_chars),
|
|
681
|
+
):
|
|
682
|
+
if value is not None:
|
|
683
|
+
params[name] = value
|
|
684
|
+
|
|
685
|
+
try:
|
|
686
|
+
result = await self._core.call("chunk", params)
|
|
687
|
+
self._last_timing_breakdown = _extract_timing_breakdown(result)
|
|
688
|
+
raw = result.get("result")
|
|
689
|
+
if not isinstance(raw, dict):
|
|
690
|
+
raise SvoChunkingError("chunk: invalid result shape")
|
|
691
|
+
raw = _unwrap_queued_command_result(raw)
|
|
692
|
+
|
|
693
|
+
if raw.get("success") is False and raw.get("error"):
|
|
694
|
+
err = raw["error"]
|
|
695
|
+
if isinstance(err, dict) and err.get("message"):
|
|
696
|
+
raise SvoChunkingError(str(err["message"]), error=err)
|
|
697
|
+
raise SvoChunkingError("chunk: server reported failure")
|
|
698
|
+
|
|
699
|
+
if (
|
|
700
|
+
raw.get("mime") == "application/x-ndjson"
|
|
701
|
+
and isinstance(raw.get("text_chunk_counts"), list)
|
|
702
|
+
and (
|
|
703
|
+
isinstance(raw.get("body"), str)
|
|
704
|
+
or (
|
|
705
|
+
isinstance(raw.get("transfer_id"), str)
|
|
706
|
+
and raw.get("transfer_id", "").strip()
|
|
707
|
+
)
|
|
708
|
+
)
|
|
709
|
+
):
|
|
710
|
+
for te in raw.get("text_errors") or []:
|
|
711
|
+
if isinstance(te, dict) and te.get("index") == 0:
|
|
712
|
+
ep = te.get("error") or {}
|
|
713
|
+
if isinstance(ep, dict) and ep.get("message"):
|
|
714
|
+
raise SvoChunkingError(str(ep["message"]), error=ep)
|
|
715
|
+
raise SvoChunkingError("chunk failed for input text")
|
|
716
|
+
per_text = await _chunk_success_ndjson_per_text_lists(
|
|
717
|
+
self._core,
|
|
718
|
+
raw,
|
|
719
|
+
)
|
|
720
|
+
if not per_text:
|
|
721
|
+
raise SvoChunkingError("chunk: empty NDJSON result")
|
|
722
|
+
return per_text[0]
|
|
723
|
+
|
|
724
|
+
results_list = _extract_batch_results(raw)
|
|
725
|
+
if not isinstance(results_list, list) or not results_list:
|
|
726
|
+
raise SvoChunkingError("chunk: no results list")
|
|
727
|
+
|
|
728
|
+
first = results_list[0]
|
|
729
|
+
if not isinstance(first, dict):
|
|
730
|
+
raise SvoChunkingError("chunk: invalid item result shape")
|
|
731
|
+
if not first.get("success", False):
|
|
732
|
+
error_payload = first.get("error", {})
|
|
733
|
+
if isinstance(error_payload, dict) and error_payload.get("message"):
|
|
734
|
+
raise SvoChunkingError(
|
|
735
|
+
str(error_payload["message"]), error=error_payload
|
|
736
|
+
)
|
|
737
|
+
raise SvoChunkingError("chunk failed for input text")
|
|
738
|
+
chunks = first.get("chunks")
|
|
739
|
+
if isinstance(chunks, list):
|
|
740
|
+
if any(not isinstance(c, dict) for c in chunks):
|
|
741
|
+
raise SvoChunkingError("chunk: invalid chunk entry in batch item")
|
|
742
|
+
return _normalize_chunk_dict_list(chunks)
|
|
743
|
+
data = first.get("data", {})
|
|
744
|
+
if isinstance(data, dict):
|
|
745
|
+
fallback_chunks = data.get("chunks", [])
|
|
746
|
+
if isinstance(fallback_chunks, list):
|
|
747
|
+
if any(not isinstance(c, dict) for c in fallback_chunks):
|
|
748
|
+
raise SvoChunkingError(
|
|
749
|
+
"chunk: invalid chunk entry in batch item data"
|
|
750
|
+
)
|
|
751
|
+
return _normalize_chunk_dict_list(fallback_chunks)
|
|
752
|
+
raise SvoChunkingError("chunk: no chunks in first batch item")
|
|
753
|
+
except SvoTimeoutError:
|
|
754
|
+
raise
|
|
755
|
+
except SvoClientError:
|
|
756
|
+
raise
|
|
757
|
+
except Exception as e:
|
|
758
|
+
raise SvoConnectionError(f"Chunk failed: {e}") from e
|
|
759
|
+
|
|
760
|
+
async def chunk_batch(
|
|
761
|
+
self,
|
|
762
|
+
texts: List[str],
|
|
763
|
+
window: int = 3,
|
|
764
|
+
language: Optional[str] = None,
|
|
765
|
+
source_id: Optional[str] = None,
|
|
766
|
+
project: Optional[str] = None,
|
|
767
|
+
unit_id: Optional[str] = None,
|
|
768
|
+
chunking_version: str = "1.0",
|
|
769
|
+
use_sv: bool = False,
|
|
770
|
+
chunk_type: Optional[str] = None,
|
|
771
|
+
aggregation_mode: Optional[str] = None,
|
|
772
|
+
respect_structural_blocks: Optional[bool] = None,
|
|
773
|
+
chunk_set: Optional[str] = None,
|
|
774
|
+
model: Optional[str] = None,
|
|
775
|
+
dimension: Optional[int] = None,
|
|
776
|
+
chunk_only: Optional[bool] = None,
|
|
777
|
+
split_level: Optional[str] = None,
|
|
778
|
+
max_chars: Optional[int] = None,
|
|
779
|
+
) -> List[List[Dict[str, Any]]]:
|
|
780
|
+
if not texts or not isinstance(texts, list):
|
|
781
|
+
raise SvoValidationError("texts must be a non-empty list")
|
|
782
|
+
|
|
783
|
+
_validate_embedding_pair(model, dimension)
|
|
784
|
+
|
|
785
|
+
warn_if_unknown_chunk_set(chunk_set)
|
|
786
|
+
|
|
787
|
+
params: Dict[str, Any] = {
|
|
788
|
+
"texts": texts,
|
|
789
|
+
"window": window,
|
|
790
|
+
"use_sv": use_sv,
|
|
791
|
+
}
|
|
792
|
+
if chunk_type is not None:
|
|
793
|
+
params["type"] = chunk_type
|
|
794
|
+
if language is not None:
|
|
795
|
+
params["language"] = language
|
|
796
|
+
if source_id is not None:
|
|
797
|
+
params["source_id"] = source_id
|
|
798
|
+
if project is not None:
|
|
799
|
+
params["project"] = project
|
|
800
|
+
if unit_id is not None:
|
|
801
|
+
params["unit_id"] = unit_id
|
|
802
|
+
if chunking_version is not None:
|
|
803
|
+
params["chunking_version"] = chunking_version
|
|
804
|
+
if aggregation_mode is not None:
|
|
805
|
+
params["aggregation_mode"] = aggregation_mode
|
|
806
|
+
if respect_structural_blocks is not None:
|
|
807
|
+
params["respect_structural_blocks"] = respect_structural_blocks
|
|
808
|
+
if chunk_set is not None:
|
|
809
|
+
params["chunk_set"] = chunk_set
|
|
810
|
+
if model is not None:
|
|
811
|
+
params["model"] = model
|
|
812
|
+
params["dimension"] = dimension
|
|
813
|
+
|
|
814
|
+
for name, value in (
|
|
815
|
+
("chunk_only", chunk_only),
|
|
816
|
+
("split_level", split_level),
|
|
817
|
+
("max_chars", max_chars),
|
|
818
|
+
):
|
|
819
|
+
if value is not None:
|
|
820
|
+
params[name] = value
|
|
821
|
+
|
|
822
|
+
try:
|
|
823
|
+
result = await self._core.call("chunk", params)
|
|
824
|
+
self._last_timing_breakdown = _extract_timing_breakdown(result)
|
|
825
|
+
except SvoTimeoutError:
|
|
826
|
+
raise
|
|
827
|
+
except SvoClientError:
|
|
828
|
+
raise
|
|
829
|
+
except Exception as e:
|
|
830
|
+
raise SvoConnectionError(f"chunk failed: {e}") from e
|
|
831
|
+
|
|
832
|
+
raw = result.get("result")
|
|
833
|
+
if not isinstance(raw, dict):
|
|
834
|
+
raise SvoChunkingError("chunk: invalid result shape")
|
|
835
|
+
raw = _unwrap_queued_command_result(raw)
|
|
836
|
+
|
|
837
|
+
if raw.get("success") is False and raw.get("error"):
|
|
838
|
+
err = raw["error"]
|
|
839
|
+
# region agent log
|
|
840
|
+
try:
|
|
841
|
+
from svo_client.debug_session_log import write_debug_ndjson
|
|
842
|
+
|
|
843
|
+
write_debug_ndjson(
|
|
844
|
+
"H_client",
|
|
845
|
+
"svo_client/client.py:chunk_batch",
|
|
846
|
+
"chunk command returned top-level error",
|
|
847
|
+
{
|
|
848
|
+
"error_keys": (
|
|
849
|
+
list(err.keys()) if isinstance(err, dict) else None
|
|
850
|
+
),
|
|
851
|
+
"message": (
|
|
852
|
+
err.get("message") if isinstance(err, dict) else None
|
|
853
|
+
),
|
|
854
|
+
"code": err.get("code") if isinstance(err, dict) else None,
|
|
855
|
+
"details": (
|
|
856
|
+
err.get("details") if isinstance(err, dict) else None
|
|
857
|
+
),
|
|
858
|
+
},
|
|
859
|
+
)
|
|
860
|
+
except Exception:
|
|
861
|
+
pass
|
|
862
|
+
# endregion
|
|
863
|
+
if isinstance(err, dict) and err.get("message"):
|
|
864
|
+
raise SvoChunkingError(str(err["message"]), error=err)
|
|
865
|
+
raise SvoChunkingError("chunk: server reported failure")
|
|
866
|
+
|
|
867
|
+
if (
|
|
868
|
+
raw.get("mime") == "application/x-ndjson"
|
|
869
|
+
and isinstance(raw.get("text_chunk_counts"), list)
|
|
870
|
+
and (
|
|
871
|
+
isinstance(raw.get("body"), str)
|
|
872
|
+
or (
|
|
873
|
+
isinstance(raw.get("transfer_id"), str)
|
|
874
|
+
and raw.get("transfer_id", "").strip()
|
|
875
|
+
)
|
|
876
|
+
)
|
|
877
|
+
):
|
|
878
|
+
return await _chunk_success_ndjson_per_text_lists(self._core, raw)
|
|
879
|
+
|
|
880
|
+
results_list = _extract_batch_results(raw)
|
|
881
|
+
if not isinstance(results_list, list):
|
|
882
|
+
raise SvoChunkingError("chunk: no results list")
|
|
883
|
+
|
|
884
|
+
out: List[List[Dict[str, Any]]] = []
|
|
885
|
+
for item in results_list:
|
|
886
|
+
if isinstance(item, dict) and item.get("success"):
|
|
887
|
+
data = item.get("data", {})
|
|
888
|
+
chunks = data.get("chunks", []) if isinstance(data, dict) else []
|
|
889
|
+
if isinstance(chunks, list):
|
|
890
|
+
if any(not isinstance(c, dict) for c in chunks):
|
|
891
|
+
raise SvoChunkingError(
|
|
892
|
+
"chunk_batch: invalid chunk entry in batch item"
|
|
893
|
+
)
|
|
894
|
+
out.append(_normalize_chunk_dict_list(chunks))
|
|
895
|
+
else:
|
|
896
|
+
out.append([])
|
|
897
|
+
else:
|
|
898
|
+
out.append([])
|
|
899
|
+
return out
|
|
900
|
+
|
|
901
|
+
async def health(self) -> Dict[str, Any]:
|
|
902
|
+
return await self._core.health()
|
|
903
|
+
|
|
904
|
+
async def info(self) -> Dict[str, Any]:
|
|
905
|
+
"""Return the server's complete documentation payload."""
|
|
906
|
+
return await self.call("info")
|
|
907
|
+
|
|
908
|
+
async def stanza_runtime(self, action: str) -> Dict[str, Any]:
|
|
909
|
+
"""Read or change Stanza runtime state (status, on, off)."""
|
|
910
|
+
return await self.call("stanza_runtime", {"action": action})
|
|
911
|
+
|
|
912
|
+
async def file(
|
|
913
|
+
self,
|
|
914
|
+
*,
|
|
915
|
+
filename: Optional[str] = None,
|
|
916
|
+
file_content_base64: Optional[str] = None,
|
|
917
|
+
filepath: Optional[str] = None,
|
|
918
|
+
transfer_id: Optional[str] = None,
|
|
919
|
+
filter_name: Optional[str] = None,
|
|
920
|
+
status_hook: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
|
921
|
+
) -> Dict[str, Any]:
|
|
922
|
+
"""Extract text through any of the server's three file acquisition modes."""
|
|
923
|
+
params = {
|
|
924
|
+
name: value
|
|
925
|
+
for name, value in (
|
|
926
|
+
("filename", filename),
|
|
927
|
+
("file_content_base64", file_content_base64),
|
|
928
|
+
("filepath", filepath),
|
|
929
|
+
("transfer_id", transfer_id),
|
|
930
|
+
("filter_name", filter_name),
|
|
931
|
+
)
|
|
932
|
+
if value is not None
|
|
933
|
+
}
|
|
934
|
+
return await self.call("file", params, status_hook=status_hook)
|
|
935
|
+
|
|
936
|
+
async def parse_chunk_result(
|
|
937
|
+
self, raw: Dict[str, Any]
|
|
938
|
+
) -> List[List[Dict[str, Any]]]:
|
|
939
|
+
"""Resolve inline/transfer NDJSON or legacy per-text chunk envelopes.
|
|
940
|
+
|
|
941
|
+
Per-text failures retain their place as empty sequences; inspect the
|
|
942
|
+
original text_errors for their structured error. Command errors raise
|
|
943
|
+
SvoChunkingError with the complete wire error in its error attribute.
|
|
944
|
+
"""
|
|
945
|
+
payload = raw
|
|
946
|
+
for _ in range(8):
|
|
947
|
+
if not isinstance(payload, dict):
|
|
948
|
+
raise SvoChunkingError("chunk: invalid result shape")
|
|
949
|
+
err = payload.get("error")
|
|
950
|
+
if err:
|
|
951
|
+
message = (
|
|
952
|
+
err.get("message", str(err)) if isinstance(err, dict) else str(err)
|
|
953
|
+
)
|
|
954
|
+
raise SvoChunkingError(message, error=err)
|
|
955
|
+
if payload.get("mime") == "application/x-ndjson":
|
|
956
|
+
return await _chunk_success_ndjson_per_text_lists(self._core, payload)
|
|
957
|
+
results = payload.get("results")
|
|
958
|
+
if not isinstance(results, list):
|
|
959
|
+
results = payload.get("result")
|
|
960
|
+
if isinstance(results, list):
|
|
961
|
+
out = []
|
|
962
|
+
for item in results:
|
|
963
|
+
if not isinstance(item, dict) or not item.get("success"):
|
|
964
|
+
out.append([])
|
|
965
|
+
continue
|
|
966
|
+
chunks = item.get("chunks")
|
|
967
|
+
if chunks is None:
|
|
968
|
+
chunks = (item.get("data") or {}).get("chunks", [])
|
|
969
|
+
out.append(_normalize_chunk_dict_list(chunks))
|
|
970
|
+
return out
|
|
971
|
+
nested = payload.get("result")
|
|
972
|
+
if not isinstance(nested, dict):
|
|
973
|
+
nested = payload.get("data")
|
|
974
|
+
if not isinstance(nested, dict):
|
|
975
|
+
break
|
|
976
|
+
payload = nested
|
|
977
|
+
raise SvoChunkingError("chunk: missing chunks or NDJSON payload")
|
|
978
|
+
|
|
979
|
+
async def download_transfer_to_path(
|
|
980
|
+
self,
|
|
981
|
+
transfer_id: str,
|
|
982
|
+
destination_path: str,
|
|
983
|
+
*,
|
|
984
|
+
acknowledge_after_download: bool = True,
|
|
985
|
+
) -> Any:
|
|
986
|
+
"""Download with declared-size precheck and optional checksum ACK."""
|
|
987
|
+
return await self._core.download_transfer_to_path(
|
|
988
|
+
transfer_id,
|
|
989
|
+
destination_path,
|
|
990
|
+
acknowledge_after_download=acknowledge_after_download,
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
async def get_download_session(self, transfer_id: str) -> Dict[str, Any]:
|
|
994
|
+
"""Read the adapter's declared artifact size and checksum."""
|
|
995
|
+
return await self._core.adapter_client.get_download_session(transfer_id)
|
|
996
|
+
|
|
997
|
+
async def transfer_ack(
|
|
998
|
+
self,
|
|
999
|
+
transfer_id: str,
|
|
1000
|
+
*,
|
|
1001
|
+
checksum_value: str,
|
|
1002
|
+
checksum_algorithm: str = "sha256",
|
|
1003
|
+
) -> Dict[str, Any]:
|
|
1004
|
+
"""Acknowledge a received artifact using its actual checksum."""
|
|
1005
|
+
return await self._core.adapter_client.jsonrpc_call(
|
|
1006
|
+
"transfer_ack",
|
|
1007
|
+
{
|
|
1008
|
+
"transfer_id": transfer_id,
|
|
1009
|
+
"checksum_algorithm": checksum_algorithm,
|
|
1010
|
+
"checksum_value": checksum_value,
|
|
1011
|
+
},
|
|
1012
|
+
)
|
|
1013
|
+
|
|
1014
|
+
async def rpc_call(
|
|
1015
|
+
self,
|
|
1016
|
+
command: str,
|
|
1017
|
+
params: Optional[Dict[str, Any]] = None,
|
|
1018
|
+
) -> Dict[str, Any]:
|
|
1019
|
+
"""Send one raw parameter invocation without command-session wrapping."""
|
|
1020
|
+
return await self._core.adapter_client.jsonrpc_call(command, params or {})
|
|
1021
|
+
|
|
1022
|
+
async def upload_file(
|
|
1023
|
+
self,
|
|
1024
|
+
source_path: str,
|
|
1025
|
+
filename: Optional[str] = None,
|
|
1026
|
+
compression: str = "identity",
|
|
1027
|
+
chunk_size: Optional[int] = None,
|
|
1028
|
+
on_progress: Optional[Callable[[int, int], None]] = None,
|
|
1029
|
+
) -> Any:
|
|
1030
|
+
"""Upload through the adapter and return its verified transfer receipt."""
|
|
1031
|
+
return await self._core.adapter_client.upload_file(
|
|
1032
|
+
source_path,
|
|
1033
|
+
filename=filename,
|
|
1034
|
+
compression=compression,
|
|
1035
|
+
chunk_size=chunk_size,
|
|
1036
|
+
on_progress=on_progress,
|
|
1037
|
+
)
|
|
1038
|
+
|
|
1039
|
+
async def get_openapi_schema(self) -> Dict[str, Any]:
|
|
1040
|
+
"""Return OpenAPI document from the chunker (adapter ``GET /openapi.json``)."""
|
|
1041
|
+
return await self._core.get_openapi_schema()
|
|
1042
|
+
|
|
1043
|
+
def open_ws_channel(
|
|
1044
|
+
self,
|
|
1045
|
+
receive_timeout: float = 60.0,
|
|
1046
|
+
heartbeat: float = 30.0,
|
|
1047
|
+
) -> "BidirectionalWsChannel":
|
|
1048
|
+
return self._core.open_ws_channel(
|
|
1049
|
+
receive_timeout=receive_timeout,
|
|
1050
|
+
heartbeat=heartbeat,
|
|
1051
|
+
)
|
|
1052
|
+
|
|
1053
|
+
async def file_chunk(
|
|
1054
|
+
self,
|
|
1055
|
+
params: Dict[str, Any],
|
|
1056
|
+
*,
|
|
1057
|
+
download_path: Optional[str] = None,
|
|
1058
|
+
status_hook: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
|
1059
|
+
) -> Dict[str, Any]:
|
|
1060
|
+
"""
|
|
1061
|
+
Run the server ``file_chunk`` command.
|
|
1062
|
+
|
|
1063
|
+
On success, returns the normalized data payload (``success``, ``type``,
|
|
1064
|
+
``transfer_id``, ``filename``, ``filter_name``, optional ``metadata``).
|
|
1065
|
+
When *download_path* is set, fetches the outbound JSON artifact via
|
|
1066
|
+
:meth:`ClientCore.download_transfer_to_path` (free-space precheck using
|
|
1067
|
+
declared size from the adapter download session, then ``transfer_ack`` so
|
|
1068
|
+
the server can release staging per g1).
|
|
1069
|
+
|
|
1070
|
+
Optional *status_hook* is passed to :meth:`ClientCore.call` for queued
|
|
1071
|
+
jobs (adapter WebSocket lifecycle / progress before completion).
|
|
1072
|
+
"""
|
|
1073
|
+
try:
|
|
1074
|
+
raw = await self._core.call("file_chunk", params, status_hook=status_hook)
|
|
1075
|
+
data = extract_file_chunk_success_payload(raw)
|
|
1076
|
+
if download_path:
|
|
1077
|
+
await self._core.download_transfer_to_path(
|
|
1078
|
+
data["transfer_id"],
|
|
1079
|
+
download_path,
|
|
1080
|
+
)
|
|
1081
|
+
return data
|
|
1082
|
+
except SvoTimeoutError:
|
|
1083
|
+
raise
|
|
1084
|
+
except SvoClientError:
|
|
1085
|
+
raise
|
|
1086
|
+
except SvoChunkingError:
|
|
1087
|
+
raise
|
|
1088
|
+
except Exception as e:
|
|
1089
|
+
raise SvoConnectionError(f"file_chunk failed: {e}") from e
|
|
1090
|
+
|
|
1091
|
+
async def close(self) -> None:
|
|
1092
|
+
"""Close underlying adapter client if it has a close method (e.g. HTTP session)."""
|
|
1093
|
+
adapter = self._core.adapter_client
|
|
1094
|
+
close_fn = getattr(adapter, "close", None)
|
|
1095
|
+
if close_fn is not None and callable(close_fn):
|
|
1096
|
+
result = close_fn()
|
|
1097
|
+
if asyncio.iscoroutine(result):
|
|
1098
|
+
await result
|
|
1099
|
+
|
|
1100
|
+
async def __aenter__(self) -> "SvoChunkerClient":
|
|
1101
|
+
return self
|
|
1102
|
+
|
|
1103
|
+
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
1104
|
+
await self.close()
|