sie-sdk 0.7.1__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/.gitignore +2 -36
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/CONTRIBUTING.md +2 -2
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/PKG-INFO +4 -2
- sie_sdk-0.7.2/README.md +202 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/pyproject.toml +3 -1
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/__init__.py +21 -2
- sie_sdk-0.7.2/src/sie_sdk/_msgpack.py +132 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/cache.py +5 -5
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/__init__.py +16 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/_shared.py +384 -44
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/async_.py +347 -53
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/errors.py +201 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/sync.py +434 -80
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/jobs.py +69 -12
- sie_sdk-0.7.2/src/sie_sdk/redaction.py +92 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/scoring.py +81 -9
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/storage.py +402 -8
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/types.py +184 -5
- sie_sdk-0.7.2/tests/client/test_backpressure_billing.py +359 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_base_url_headers.py +51 -2
- sie_sdk-0.7.2/tests/client/test_base_url_validation.py +55 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_chat.py +141 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_chat_async.py +119 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_generate.py +180 -23
- sie_sdk-0.7.2/tests/client/test_incomplete_batch.py +198 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_jobs.py +301 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_malformed_retry_after.py +13 -11
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_model_load_failed.py +1 -1
- sie_sdk-0.7.2/tests/client/test_recommend.py +189 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_responses.py +40 -0
- sie_sdk-0.7.2/tests/client/test_stream_timeout.py +135 -0
- sie_sdk-0.7.2/tests/client/test_terminal_msgpack.py +175 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_transport_error_retry.py +130 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_cache.py +69 -0
- sie_sdk-0.7.2/tests/test_error_exports.py +145 -0
- sie_sdk-0.7.2/tests/test_msgpack_safety.py +103 -0
- sie_sdk-0.7.2/tests/test_redaction.py +78 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_scoring.py +83 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_storage.py +285 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_storage_copy.py +57 -1
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_storage_write.py +70 -1
- sie_sdk-0.7.2/tests/test_wire_contract.py +125 -0
- sie_sdk-0.7.1/README.md +0 -117
- sie_sdk-0.7.1/src/sie_sdk/redaction.py +0 -41
- sie_sdk-0.7.1/tests/test_redaction.py +0 -33
- sie_sdk-0.7.1/tests/test_wire_contract.py +0 -64
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/LICENSE +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/audio.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/bundle_utils.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/client/_sse.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/documents.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/encoding.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/exceptions.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/files.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/images.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/src/sie_sdk/queue_types.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/__init__.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/conftest.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_async.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_audio_extract.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_batches.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_connections.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_embeddings_error_envelope.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_error_request_metadata.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_estimate.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_files.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_gateway_timeout_retry.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_input_too_long.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_no_engine_surface.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_oom_retry.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_provisioning.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_retry_jitter.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_settled_charge.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_sse.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_sync.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_timeout.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/client/test_version_skew.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_audio.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_bundle_utils.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_documents.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_exceptions.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_images.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_jobs_body.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_sdk_smoke.py +0 -0
- {sie_sdk-0.7.1 → sie_sdk-0.7.2}/tests/test_worker_id_normalization.py +0 -0
|
@@ -142,10 +142,8 @@ celerybeat.pid
|
|
|
142
142
|
|
|
143
143
|
# Environments
|
|
144
144
|
.env
|
|
145
|
-
#
|
|
146
|
-
|
|
147
|
-
# repo (audit §1/§14). Belt-and-braces so a local copy can never be committed.
|
|
148
|
-
*.sie-cloud-secrets.env
|
|
145
|
+
# Local service credential bundles must never be committed.
|
|
146
|
+
*-secrets.env
|
|
149
147
|
.env.cloud
|
|
150
148
|
.venv
|
|
151
149
|
env/
|
|
@@ -244,33 +242,6 @@ override.tf.json
|
|
|
244
242
|
# tfvars files may contain secrets
|
|
245
243
|
*.tfvars
|
|
246
244
|
*.tfvars.json
|
|
247
|
-
# Reviewed worker-AMI release contracts contain only immutable public package,
|
|
248
|
-
# AMI, and Terraform backend coordinates; the release workflow accepts only
|
|
249
|
-
# committed files from this exact internal path.
|
|
250
|
-
!deploy/terraform/aws/internal-examples/cloud-benchmark-runner/worker-ami/releases/*.tfvars.json
|
|
251
|
-
# Exception: this edge tfvars carries only a Secrets Manager ARN (a resource
|
|
252
|
-
# identifier, not a secret) so the staging-us edge apply needs no manual -var.
|
|
253
|
-
!deploy/cloud/terraform/aws-edge/staging-us.tfvars
|
|
254
|
-
# This dev adoption contract contains only public AWS resource identifiers; it
|
|
255
|
-
# is checked in so a reviewed plan cannot silently target a different orphan
|
|
256
|
-
# VPC/listener through an operator-local tfvars file.
|
|
257
|
-
!deploy/cloud/terraform/dev-controlplane-https-adoption/dev-us.tfvars
|
|
258
|
-
!deploy/cloud/terraform/dev-controlplane-https-adoption/.terraform.lock.hcl
|
|
259
|
-
# Managed release roots commit their exact provider selections. Release
|
|
260
|
-
# preflights and applies use these read-only instead of selecting a newer
|
|
261
|
-
# compatible provider during a long-running deployment.
|
|
262
|
-
!deploy/cloud/terraform/release-infrastructure/.terraform.lock.hcl
|
|
263
|
-
!deploy/cloud/terraform/aws-controlplane/.terraform.lock.hcl
|
|
264
|
-
!deploy/cloud/terraform/aws-edge/.terraform.lock.hcl
|
|
265
|
-
!deploy/cloud/terraform/s3-state/.terraform.lock.hcl
|
|
266
|
-
!deploy/cloud/terraform/vercel/.terraform.lock.hcl
|
|
267
|
-
!deploy/cloud/terraform/console-preview/.terraform.lock.hcl
|
|
268
|
-
!deploy/cloud/terraform/examples/dev/.terraform.lock.hcl
|
|
269
|
-
!deploy/cloud/terraform/examples/dev-eu/.terraform.lock.hcl
|
|
270
|
-
!deploy/cloud/terraform/examples/staging/.terraform.lock.hcl
|
|
271
|
-
!deploy/cloud/terraform/examples/staging-eu/.terraform.lock.hcl
|
|
272
|
-
!deploy/cloud/terraform/examples/prod-us/.terraform.lock.hcl
|
|
273
|
-
# Keep .terraform.lock.hcl for reproducibility (provider versions)
|
|
274
245
|
|
|
275
246
|
# Node.js
|
|
276
247
|
node_modules/
|
|
@@ -294,12 +265,7 @@ tmp/
|
|
|
294
265
|
.tmp/
|
|
295
266
|
.local/
|
|
296
267
|
|
|
297
|
-
.requirements-modal.txt
|
|
298
|
-
.mpa-i6pn-*-requirements-modal.txt
|
|
299
268
|
b6_results.json
|
|
300
|
-
# Raw dev-cloud load-test reports are local artifacts, not source-controlled baselines.
|
|
301
|
-
packages/sie_cloud/bench/results/dev-us-*/
|
|
302
|
-
|
|
303
269
|
# Personal mise overrides (per-developer toolchain tweaks, e.g. rustup
|
|
304
270
|
# component naming differences across rustup versions).
|
|
305
271
|
mise.local.toml
|
|
@@ -14,12 +14,12 @@ mise trust && mise install
|
|
|
14
14
|
## Running Tests
|
|
15
15
|
|
|
16
16
|
```bash
|
|
17
|
-
mise run test packages/sie_sdk
|
|
17
|
+
mise run test -- packages/sie_sdk
|
|
18
18
|
```
|
|
19
19
|
|
|
20
20
|
## Code Style
|
|
21
21
|
|
|
22
|
-
- Format with `mise run lint -f`
|
|
22
|
+
- Format with `mise run lint -- -f`
|
|
23
23
|
- Type check with `mise run typecheck`
|
|
24
24
|
|
|
25
25
|
## Pull Requests
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: sie-sdk
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Search Inference Engine - Python SDK
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -20,8 +20,10 @@ Requires-Dist: langchain-core<1,>=0.3; extra == 'langchain'
|
|
|
20
20
|
Provides-Extra: llamaindex
|
|
21
21
|
Requires-Dist: llama-index-core<1,>=0.12; extra == 'llamaindex'
|
|
22
22
|
Provides-Extra: storage
|
|
23
|
+
Requires-Dist: alibabacloud-credentials<2,>=1.0.12; extra == 'storage'
|
|
23
24
|
Requires-Dist: azure-identity<2,>=1.17; extra == 'storage'
|
|
24
25
|
Requires-Dist: azure-storage-blob<13,>=12.20; extra == 'storage'
|
|
25
26
|
Requires-Dist: boto3<2,>=1.35; extra == 'storage'
|
|
26
27
|
Requires-Dist: google-cloud-storage<3,>=2.18; extra == 'storage'
|
|
27
28
|
Requires-Dist: huggingface-hub[hf-transfer]<2,>=0.26; extra == 'storage'
|
|
29
|
+
Requires-Dist: oss2<3,>=2.19.1; extra == 'storage'
|
sie_sdk-0.7.2/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# SIE SDK
|
|
2
|
+
|
|
3
|
+
Python client SDK for the SIE inference server.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install sie-sdk
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from sie_sdk import SIEClient
|
|
15
|
+
from sie_sdk.types import Item
|
|
16
|
+
|
|
17
|
+
client = SIEClient("http://localhost:8080")
|
|
18
|
+
|
|
19
|
+
# Encode text. Results are TypedDicts — access fields by key.
|
|
20
|
+
result = client.encode("BAAI/bge-m3", Item(text="Hello world"))
|
|
21
|
+
print(result["dense"].shape) # (1024,)
|
|
22
|
+
|
|
23
|
+
# Score items against a query with a reranker model.
|
|
24
|
+
# Scores come back sorted by relevance (rank 0 = most relevant).
|
|
25
|
+
scores = client.score(
|
|
26
|
+
"BAAI/bge-reranker-v2-m3",
|
|
27
|
+
query=Item(text="What is machine learning?"),
|
|
28
|
+
items=[
|
|
29
|
+
Item(id="doc-1", text="Machine learning is a subfield of AI."),
|
|
30
|
+
Item(id="doc-2", text="Python is a programming language."),
|
|
31
|
+
],
|
|
32
|
+
)
|
|
33
|
+
for entry in scores["scores"]:
|
|
34
|
+
print(entry["item_id"], entry["score"])
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Connecting to a managed SIE platform
|
|
38
|
+
|
|
39
|
+
The examples above target a local server. For a managed SIE gateway,
|
|
40
|
+
pass the gateway URL as `base_url` and your API key (sent as a Bearer
|
|
41
|
+
token):
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from sie_sdk import SIEClient
|
|
45
|
+
|
|
46
|
+
client = SIEClient(
|
|
47
|
+
"https://your-gateway.example.com",
|
|
48
|
+
api_key="YOUR_API_KEY",
|
|
49
|
+
)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Object storage and model caches
|
|
53
|
+
|
|
54
|
+
Install the `storage` extra to use `s3://`, `gs://`, `abfs(s)://`, or native
|
|
55
|
+
Alibaba `oss://` model/cache paths:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install 'sie-sdk[storage]'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Alibaba OSS always uses region-scoped Signature V4. Set `SIE_OSS_REGION` to
|
|
62
|
+
the bucket's region (for example, `eu-central-1`). Set
|
|
63
|
+
`SIE_OSS_USE_INTERNAL_ENDPOINT=true` only inside the matching Alibaba Cloud
|
|
64
|
+
network; the SDK derives the HTTPS endpoint and does not accept endpoint URLs
|
|
65
|
+
from storage paths.
|
|
66
|
+
|
|
67
|
+
On ACK, RRSA supplies `ALIBABA_CLOUD_ROLE_ARN`,
|
|
68
|
+
`ALIBABA_CLOUD_OIDC_PROVIDER_ARN`, and `ALIBABA_CLOUD_OIDC_TOKEN_FILE`. The SDK
|
|
69
|
+
requires all three together and refreshes short-lived credentials through the
|
|
70
|
+
Alibaba Credentials client. Local operators can use that client's standard
|
|
71
|
+
credential sources instead. Never place credentials, queries, or fragments in
|
|
72
|
+
an `oss://` URL.
|
|
73
|
+
|
|
74
|
+
`oss://` is supported for model discovery, cache population, and object copies.
|
|
75
|
+
It is deliberately not a mutable `sie-config` epoch store because OSS
|
|
76
|
+
PutObject cannot provide the required non-empty ETag compare-and-swap contract;
|
|
77
|
+
use local/PVC, S3, GCS, or Azure storage for that store.
|
|
78
|
+
|
|
79
|
+
## Error handling
|
|
80
|
+
|
|
81
|
+
Server-reported errors are raised as typed exceptions from
|
|
82
|
+
`sie_sdk.client.errors`; all inherit from `SIEError`, and errors that
|
|
83
|
+
carry a server response expose `.code` and `.status_code`. Invalid
|
|
84
|
+
client-side arguments (for example, a bad `base_url_headers` value)
|
|
85
|
+
raise ordinary Python exceptions such as `ValueError`.
|
|
86
|
+
|
|
87
|
+
Several `503` codes are transient and retried automatically (see the
|
|
88
|
+
next section for the `RESOURCE_EXHAUSTED` budget):
|
|
89
|
+
|
|
90
|
+
- `PROVISIONING` — the cluster is scaling capacity from zero. Retry is
|
|
91
|
+
governed by `wait_for_capacity`: retried under `provision_timeout_s`
|
|
92
|
+
when `True` (the default); surfaces immediately as
|
|
93
|
+
`ProvisioningError` when `False`.
|
|
94
|
+
- `MODEL_LOADING` — the worker accepted the request and is cold-loading
|
|
95
|
+
the target model. Retried until `provision_timeout_s`; raises
|
|
96
|
+
`ModelLoadingError` if the budget is exhausted.
|
|
97
|
+
- `LORA_LOADING` — the requested LoRA adapter is still loading. Retried
|
|
98
|
+
a bounded number of times; raises `LoraLoadingError` when the retry
|
|
99
|
+
budget is exhausted.
|
|
100
|
+
- `RESOURCE_EXHAUSTED` — the server ran out of GPU memory and exhausted
|
|
101
|
+
its internal recovery. Retried with bounded backoff; raises
|
|
102
|
+
`ResourceExhaustedError` when retries run out. Pass
|
|
103
|
+
`max_oom_retries=0` to disable these retries and fail fast.
|
|
104
|
+
|
|
105
|
+
One generation-specific error is terminal and never retried:
|
|
106
|
+
|
|
107
|
+
- `empty_model_output` — the generation finished nominally but produced
|
|
108
|
+
no visible output text (for example, private reasoning consumed the
|
|
109
|
+
whole token budget). Tokens were genuinely consumed, so the request
|
|
110
|
+
is not re-run. Surfaces as `ServerError` with
|
|
111
|
+
`code == "empty_model_output"`; on streaming calls it is raised
|
|
112
|
+
mid-stream with the gateway request id attached for correlation.
|
|
113
|
+
|
|
114
|
+
## Handling resource exhaustion
|
|
115
|
+
|
|
116
|
+
The SDK automatically retries requests that the server signals as
|
|
117
|
+
transient — model still loading, scale-from-zero in progress, or **GPU
|
|
118
|
+
memory pressure (`RESOURCE_EXHAUSTED`)**. You don't have to write
|
|
119
|
+
retry logic for these.
|
|
120
|
+
|
|
121
|
+
### What happens by default
|
|
122
|
+
|
|
123
|
+
When the server's GPU runs out of memory mid-request, the worker first
|
|
124
|
+
attempts an internal recovery (clear cache → evict an idle sibling
|
|
125
|
+
model → recursively halve the batch). If that succeeds you get a normal
|
|
126
|
+
200 response — slightly slower than usual.
|
|
127
|
+
|
|
128
|
+
If recovery is exhausted, the server returns `503 RESOURCE_EXHAUSTED`
|
|
129
|
+
with a `Retry-After: 5` header. The SDK then retries with bounded
|
|
130
|
+
exponential backoff (5s → 10s → 20s, capped at 30s, max 3 attempts).
|
|
131
|
+
The first retry logs at WARNING so you can see it at default log
|
|
132
|
+
levels:
|
|
133
|
+
|
|
134
|
+
```text
|
|
135
|
+
WARNING sie_sdk.client.sync: Server resource exhausted, retrying in 5.0s (attempt 1/3, elapsed: 0.4s, timeout: 900.0s)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
If all retries are exhausted, the SDK raises
|
|
139
|
+
`sie_sdk.client.errors.ResourceExhaustedError` (a subclass of
|
|
140
|
+
`ServerError`).
|
|
141
|
+
|
|
142
|
+
### Tuning the behaviour
|
|
143
|
+
|
|
144
|
+
| Parameter | Default | Effect |
|
|
145
|
+
|--|--|--|
|
|
146
|
+
| `max_oom_retries=N` | `3` | Cap on auto-retries. Pass `0` to fail fast. |
|
|
147
|
+
| `provision_timeout_s=T` | `900` (15 min) | Total wall-clock budget. OOM retries are clamped to the remaining budget — you'll never sleep past your timeout. |
|
|
148
|
+
|
|
149
|
+
### Examples
|
|
150
|
+
|
|
151
|
+
**Default (resilient) — recommended for most callers:**
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
result = client.encode("BAAI/bge-m3", Item(text="Hello"))
|
|
155
|
+
# Auto-retries on RESOURCE_EXHAUSTED. May take up to ~35s extra
|
|
156
|
+
# if recovery + retries are needed.
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Fail-fast (CI tests, latency-critical hot paths):**
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from sie_sdk.client.errors import ResourceExhaustedError
|
|
163
|
+
|
|
164
|
+
try:
|
|
165
|
+
result = client.encode(
|
|
166
|
+
"BAAI/bge-m3",
|
|
167
|
+
Item(text="Hello"),
|
|
168
|
+
max_oom_retries=0, # No retries; surface failure immediately
|
|
169
|
+
)
|
|
170
|
+
except ResourceExhaustedError:
|
|
171
|
+
# Server is under memory pressure — fall back to a smaller model,
|
|
172
|
+
# batch later, or surface to the user.
|
|
173
|
+
...
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
**Tight wall-clock budget:**
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
result = client.encode(
|
|
180
|
+
"BAAI/bge-m3",
|
|
181
|
+
Item(text="Hello"),
|
|
182
|
+
provision_timeout_s=10.0, # Total budget; OOM retries clamped to it
|
|
183
|
+
)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### What you'll see in your logs
|
|
187
|
+
|
|
188
|
+
| Server state | Client outcome | Log level |
|
|
189
|
+
|--|--|--|
|
|
190
|
+
| GPU OK | 200, normal latency | (none) |
|
|
191
|
+
| OOM, server-side recovery succeeds | 200, +1-3s latency | (none) |
|
|
192
|
+
| OOM, SDK retries succeed | 200, +5-35s latency | WARNING on 1st retry |
|
|
193
|
+
| OOM, SDK retries exhausted | `ResourceExhaustedError` | WARNING + traceback |
|
|
194
|
+
|
|
195
|
+
If you see frequent `Server resource exhausted, retrying...` warnings,
|
|
196
|
+
your cluster's GPU pool is undersized for the workload. Talk to the
|
|
197
|
+
operator running SIE — they have observability and tuning knobs
|
|
198
|
+
(`SIE_OOM_RECOVERY__*`) that aren't visible from the SDK side.
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
Apache 2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sie-sdk"
|
|
3
|
-
version = "0.7.
|
|
3
|
+
version = "0.7.2"
|
|
4
4
|
description = "Search Inference Engine - Python SDK"
|
|
5
5
|
requires-python = ">=3.12"
|
|
6
6
|
license = { text = "Apache-2.0" }
|
|
@@ -27,11 +27,13 @@ dependencies = [
|
|
|
27
27
|
|
|
28
28
|
[project.optional-dependencies]
|
|
29
29
|
storage = [
|
|
30
|
+
"alibabacloud-credentials>=1.0.12,<2",
|
|
30
31
|
"azure-identity>=1.17,<2",
|
|
31
32
|
"azure-storage-blob>=12.20,<13",
|
|
32
33
|
"boto3>=1.35,<2",
|
|
33
34
|
"google-cloud-storage>=2.18,<3",
|
|
34
35
|
"huggingface-hub[hf-transfer]>=0.26,<2",
|
|
36
|
+
"oss2>=2.19.1,<3",
|
|
35
37
|
]
|
|
36
38
|
langchain = ["langchain-core>=0.3,<1"]
|
|
37
39
|
llamaindex = ["llama-index-core>=0.12,<1"]
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Example:
|
|
4
4
|
>>> from sie_sdk import SIEClient
|
|
5
5
|
>>> client = SIEClient("http://localhost:8080")
|
|
6
|
-
>>> result = client.encode("bge-m3", {"text": "Hello world"})
|
|
6
|
+
>>> result = client.encode("BAAI/bge-m3", {"text": "Hello world"})
|
|
7
7
|
>>> result["dense"] # np.ndarray, shape [1024]
|
|
8
8
|
|
|
9
9
|
For ColBERT/late interaction models, use the scoring module:
|
|
@@ -12,19 +12,27 @@ For ColBERT/late interaction models, use the scoring module:
|
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
from sie_sdk.client import (
|
|
15
|
+
AccountInactiveError,
|
|
16
|
+
AccountStateUnavailableError,
|
|
15
17
|
EstimateUnroutableError,
|
|
18
|
+
IncompleteBatchError,
|
|
16
19
|
InputTooLongError,
|
|
20
|
+
InsufficientCreditsError,
|
|
21
|
+
JobFailedError,
|
|
17
22
|
LoraLoadingError,
|
|
18
23
|
ModelLoadFailedError,
|
|
19
24
|
ModelLoadingError,
|
|
20
25
|
PoolError,
|
|
21
26
|
ProvisioningError,
|
|
27
|
+
RateLimitError,
|
|
22
28
|
RequestError,
|
|
29
|
+
ResourceExhaustedError,
|
|
23
30
|
ServerError,
|
|
24
31
|
SIEAsyncClient,
|
|
25
32
|
SIEClient,
|
|
26
33
|
SIEConnectionError,
|
|
27
34
|
SIEError,
|
|
35
|
+
SpendLimitError,
|
|
28
36
|
)
|
|
29
37
|
from sie_sdk.encoding import (
|
|
30
38
|
SparseVector,
|
|
@@ -34,8 +42,9 @@ from sie_sdk.encoding import (
|
|
|
34
42
|
sparse_embedding,
|
|
35
43
|
sparse_embedding_dict,
|
|
36
44
|
)
|
|
45
|
+
from sie_sdk.exceptions import GatedModelError
|
|
37
46
|
from sie_sdk.files import resolve_upload
|
|
38
|
-
from sie_sdk.jobs import build_job_body, connection_name
|
|
47
|
+
from sie_sdk.jobs import MalformedChunkError, build_job_body, connection_name
|
|
39
48
|
from sie_sdk.types import (
|
|
40
49
|
# Response types
|
|
41
50
|
AppliedRate,
|
|
@@ -93,6 +102,8 @@ from sie_sdk.types import (
|
|
|
93
102
|
__version__ = "0.1.0"
|
|
94
103
|
|
|
95
104
|
__all__ = [
|
|
105
|
+
"AccountInactiveError",
|
|
106
|
+
"AccountStateUnavailableError",
|
|
96
107
|
"AppliedRate",
|
|
97
108
|
"AssignedWorkerInfo",
|
|
98
109
|
"Batch",
|
|
@@ -116,13 +127,17 @@ __all__ = [
|
|
|
116
127
|
"File",
|
|
117
128
|
"FileDeleted",
|
|
118
129
|
"FileList",
|
|
130
|
+
"GatedModelError",
|
|
119
131
|
"GenerateChunk",
|
|
120
132
|
"GenerateResult",
|
|
121
133
|
"GenerationUsage",
|
|
122
134
|
"HealthResponse",
|
|
135
|
+
"IncompleteBatchError",
|
|
123
136
|
"InputTooLongError",
|
|
137
|
+
"InsufficientCreditsError",
|
|
124
138
|
"Item",
|
|
125
139
|
"JobChunk",
|
|
140
|
+
"JobFailedError",
|
|
126
141
|
"JobFieldMap",
|
|
127
142
|
"JobList",
|
|
128
143
|
"JobResultItem",
|
|
@@ -131,6 +146,7 @@ __all__ = [
|
|
|
131
146
|
"JobStatus",
|
|
132
147
|
"JobSubmitResult",
|
|
133
148
|
"LoraLoadingError",
|
|
149
|
+
"MalformedChunkError",
|
|
134
150
|
"ModelInfo",
|
|
135
151
|
"ModelLoadFailedError",
|
|
136
152
|
"ModelLoadingError",
|
|
@@ -144,7 +160,9 @@ __all__ = [
|
|
|
144
160
|
"PoolStatusInfo",
|
|
145
161
|
"ProvisioningError",
|
|
146
162
|
"RateIdentity",
|
|
163
|
+
"RateLimitError",
|
|
147
164
|
"RequestError",
|
|
165
|
+
"ResourceExhaustedError",
|
|
148
166
|
"SIEAsyncClient",
|
|
149
167
|
"SIEClient",
|
|
150
168
|
"SIEConnectionError",
|
|
@@ -153,6 +171,7 @@ __all__ = [
|
|
|
153
171
|
"ServerError",
|
|
154
172
|
"SparseResult",
|
|
155
173
|
"SparseVector",
|
|
174
|
+
"SpendLimitError",
|
|
156
175
|
"StatusMessage",
|
|
157
176
|
"TimingInfo",
|
|
158
177
|
"WorkerInfo",
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Numeric-only MessagePack helpers for SDK request and response bodies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from importlib import import_module
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
_native_msgpack = import_module("msgpack._cmsgpack")
|
|
13
|
+
except ImportError: # pragma: no cover - msgpack wheels use the C extension.
|
|
14
|
+
_native_msgpack = import_module("msgpack.fallback")
|
|
15
|
+
|
|
16
|
+
_NativePacker = _native_msgpack.Packer
|
|
17
|
+
_native_unpackb = _native_msgpack.unpackb
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
_NUMERIC_DTYPE_CODES = frozenset(
|
|
21
|
+
{
|
|
22
|
+
"<f2",
|
|
23
|
+
"<f4",
|
|
24
|
+
"<f8",
|
|
25
|
+
"<i2",
|
|
26
|
+
"<i4",
|
|
27
|
+
"<i8",
|
|
28
|
+
"<u2",
|
|
29
|
+
"<u4",
|
|
30
|
+
"<u8",
|
|
31
|
+
">f2",
|
|
32
|
+
">f4",
|
|
33
|
+
">f8",
|
|
34
|
+
">i2",
|
|
35
|
+
">i4",
|
|
36
|
+
">i8",
|
|
37
|
+
">u2",
|
|
38
|
+
">u4",
|
|
39
|
+
">u8",
|
|
40
|
+
"|b1",
|
|
41
|
+
"|i1",
|
|
42
|
+
"|u1",
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
_MAX_NUMPY_DIMENSIONS = 8
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class UnsafeMessagePackError(ValueError):
|
|
49
|
+
"""A MessagePack NumPy sentinel is outside the SDK's numeric wire contract."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _numeric_dtype(value: object) -> np.dtype[Any]:
|
|
53
|
+
if not isinstance(value, str) or value not in _NUMERIC_DTYPE_CODES:
|
|
54
|
+
raise UnsafeMessagePackError("MessagePack NumPy dtype is not an allowed numeric dtype")
|
|
55
|
+
dtype = np.dtype(value)
|
|
56
|
+
if dtype.hasobject or dtype.fields is not None or dtype.subdtype is not None:
|
|
57
|
+
raise UnsafeMessagePackError("MessagePack NumPy dtype is not a flat numeric dtype")
|
|
58
|
+
return dtype
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _numeric_default(value: object) -> Mapping[bytes, object]:
|
|
62
|
+
if isinstance(value, np.ndarray):
|
|
63
|
+
dtype = _numeric_dtype(value.dtype.str)
|
|
64
|
+
contiguous = np.ascontiguousarray(value, dtype=dtype)
|
|
65
|
+
return {
|
|
66
|
+
b"nd": True,
|
|
67
|
+
b"type": dtype.str,
|
|
68
|
+
b"kind": b"",
|
|
69
|
+
b"shape": contiguous.shape,
|
|
70
|
+
b"data": contiguous.tobytes(),
|
|
71
|
+
}
|
|
72
|
+
if isinstance(value, np.generic):
|
|
73
|
+
dtype = _numeric_dtype(value.dtype.str)
|
|
74
|
+
return {
|
|
75
|
+
b"nd": False,
|
|
76
|
+
b"type": dtype.str,
|
|
77
|
+
b"data": value.tobytes(),
|
|
78
|
+
}
|
|
79
|
+
raise TypeError(f"cannot MessagePack-encode {type(value).__name__}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def packb(value: object, *, use_bin_type: bool = True) -> bytes:
|
|
83
|
+
"""Pack one SDK body without installing process-global NumPy hooks."""
|
|
84
|
+
return _NativePacker(default=_numeric_default, use_bin_type=use_bin_type).pack(value)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _numeric_object_hook(value: dict[object, object]) -> object:
|
|
88
|
+
if b"nd" not in value:
|
|
89
|
+
if b"complex" in value:
|
|
90
|
+
raise UnsafeMessagePackError("complex MessagePack NumPy values are not supported")
|
|
91
|
+
return value
|
|
92
|
+
|
|
93
|
+
is_array = value.get(b"nd")
|
|
94
|
+
if not isinstance(is_array, bool):
|
|
95
|
+
raise UnsafeMessagePackError("MessagePack NumPy nd marker must be boolean")
|
|
96
|
+
if value.get(b"kind", b"") not in {b"", ""}:
|
|
97
|
+
# In particular, kind=O invokes pickle.loads in msgpack-numpy and
|
|
98
|
+
# kind=V admits attacker-selected structured dtype descriptors.
|
|
99
|
+
raise UnsafeMessagePackError("object and structured MessagePack NumPy values are not supported")
|
|
100
|
+
|
|
101
|
+
dtype = _numeric_dtype(value.get(b"type"))
|
|
102
|
+
data = value.get(b"data")
|
|
103
|
+
if not isinstance(data, bytes):
|
|
104
|
+
raise UnsafeMessagePackError("MessagePack NumPy data must be bytes")
|
|
105
|
+
|
|
106
|
+
if not is_array:
|
|
107
|
+
if len(data) != dtype.itemsize:
|
|
108
|
+
raise UnsafeMessagePackError("MessagePack NumPy scalar byte length does not match its dtype")
|
|
109
|
+
return np.frombuffer(data, dtype=dtype, count=1)[0]
|
|
110
|
+
|
|
111
|
+
raw_shape = value.get(b"shape")
|
|
112
|
+
if not isinstance(raw_shape, list) or len(raw_shape) > _MAX_NUMPY_DIMENSIONS:
|
|
113
|
+
raise UnsafeMessagePackError("MessagePack NumPy array shape is malformed")
|
|
114
|
+
shape: list[int] = []
|
|
115
|
+
elements = 1
|
|
116
|
+
for raw_dimension in raw_shape:
|
|
117
|
+
if not isinstance(raw_dimension, int) or isinstance(raw_dimension, bool) or raw_dimension < 0:
|
|
118
|
+
raise UnsafeMessagePackError("MessagePack NumPy array dimension is malformed")
|
|
119
|
+
dimension = int(raw_dimension)
|
|
120
|
+
elements *= dimension
|
|
121
|
+
shape.append(dimension)
|
|
122
|
+
if elements * dtype.itemsize != len(data):
|
|
123
|
+
raise UnsafeMessagePackError("MessagePack NumPy array byte length does not match its shape")
|
|
124
|
+
return np.frombuffer(data, dtype=dtype).reshape(tuple(shape))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def unpackb(value: bytes, *, numeric_arrays: bool) -> Any:
|
|
128
|
+
"""Unpack one untrusted body with optional numeric-only NumPy decoding."""
|
|
129
|
+
kwargs: dict[str, Any] = {"raw": False}
|
|
130
|
+
if numeric_arrays:
|
|
131
|
+
kwargs["object_hook"] = _numeric_object_hook
|
|
132
|
+
return _native_unpackb(value, **kwargs)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Implements the caching hierarchy for model weights:
|
|
4
4
|
1. Local cache (HF_HOME/hub by default)
|
|
5
|
-
2. Cluster cache (S3/GCS/Azure object storage)
|
|
5
|
+
2. Cluster cache (S3/GCS/Azure/Alibaba OSS object storage)
|
|
6
6
|
3. HuggingFace Hub fallback (if enabled)
|
|
7
7
|
|
|
8
8
|
The caching is transparent to adapters - they always see files in local cache.
|
|
@@ -51,7 +51,7 @@ class CacheConfig:
|
|
|
51
51
|
"""Local cache directory (usually HF_HOME/hub)."""
|
|
52
52
|
|
|
53
53
|
cluster_cache: str | None = None
|
|
54
|
-
"""Cluster cache URL (s3://, gs://, abfs://, or
|
|
54
|
+
"""Cluster cache URL (s3://, gs://, abfs(s)://, or oss://), or None if not configured."""
|
|
55
55
|
|
|
56
56
|
hf_fallback: bool = True
|
|
57
57
|
"""Whether to fallback to HuggingFace Hub for downloads."""
|
|
@@ -62,7 +62,7 @@ def get_cache_config() -> CacheConfig:
|
|
|
62
62
|
|
|
63
63
|
Reads:
|
|
64
64
|
SIE_LOCAL_CACHE: Local cache directory (default: HF_HOME/hub)
|
|
65
|
-
SIE_CLUSTER_CACHE: Cluster cache URL (s3://, gs://, abfs://, or
|
|
65
|
+
SIE_CLUSTER_CACHE: Cluster cache URL (s3://, gs://, abfs(s)://, or oss://)
|
|
66
66
|
SIE_HF_FALLBACK: Whether to enable HF Hub fallback (default: true)
|
|
67
67
|
|
|
68
68
|
Returns:
|
|
@@ -79,7 +79,7 @@ def get_cache_config() -> CacheConfig:
|
|
|
79
79
|
else:
|
|
80
80
|
local_cache = Path.home() / ".cache" / "huggingface" / "hub"
|
|
81
81
|
|
|
82
|
-
# Cluster cache: S3/GCS/Azure object-store URL
|
|
82
|
+
# Cluster cache: S3/GCS/Azure/Alibaba OSS object-store URL
|
|
83
83
|
cluster_cache = os.environ.get("SIE_CLUSTER_CACHE")
|
|
84
84
|
|
|
85
85
|
# HF fallback: default true
|
|
@@ -377,7 +377,7 @@ def populate_cluster_cache(
|
|
|
377
377
|
) -> bool:
|
|
378
378
|
"""Upload model from local cache to cluster cache.
|
|
379
379
|
|
|
380
|
-
Used by
|
|
380
|
+
Used by cluster-management clients to pre-populate cluster cache. Inverse of
|
|
381
381
|
:func:`_download_from_cluster_cache`: the model's HF-style cache
|
|
382
382
|
directory (``models--{org}--{name}``) is mirrored recursively to the
|
|
383
383
|
same folder name under ``config.cluster_cache``, so workers hydrating
|
|
@@ -5,32 +5,48 @@ Re-exports all client classes and errors for backwards compatibility.
|
|
|
5
5
|
|
|
6
6
|
from sie_sdk.client.async_ import SIEAsyncClient
|
|
7
7
|
from sie_sdk.client.errors import (
|
|
8
|
+
AccountInactiveError,
|
|
9
|
+
AccountStateUnavailableError,
|
|
8
10
|
EstimateUnroutableError,
|
|
11
|
+
IncompleteBatchError,
|
|
9
12
|
InputTooLongError,
|
|
13
|
+
InsufficientCreditsError,
|
|
14
|
+
JobFailedError,
|
|
10
15
|
LoraLoadingError,
|
|
11
16
|
ModelLoadFailedError,
|
|
12
17
|
ModelLoadingError,
|
|
13
18
|
PoolError,
|
|
14
19
|
ProvisioningError,
|
|
20
|
+
RateLimitError,
|
|
15
21
|
RequestError,
|
|
22
|
+
ResourceExhaustedError,
|
|
16
23
|
ServerError,
|
|
17
24
|
SIEConnectionError,
|
|
18
25
|
SIEError,
|
|
26
|
+
SpendLimitError,
|
|
19
27
|
)
|
|
20
28
|
from sie_sdk.client.sync import SIEClient
|
|
21
29
|
|
|
22
30
|
__all__ = [
|
|
31
|
+
"AccountInactiveError",
|
|
32
|
+
"AccountStateUnavailableError",
|
|
23
33
|
"EstimateUnroutableError",
|
|
34
|
+
"IncompleteBatchError",
|
|
24
35
|
"InputTooLongError",
|
|
36
|
+
"InsufficientCreditsError",
|
|
37
|
+
"JobFailedError",
|
|
25
38
|
"LoraLoadingError",
|
|
26
39
|
"ModelLoadFailedError",
|
|
27
40
|
"ModelLoadingError",
|
|
28
41
|
"PoolError",
|
|
29
42
|
"ProvisioningError",
|
|
43
|
+
"RateLimitError",
|
|
30
44
|
"RequestError",
|
|
45
|
+
"ResourceExhaustedError",
|
|
31
46
|
"SIEAsyncClient",
|
|
32
47
|
"SIEClient",
|
|
33
48
|
"SIEConnectionError",
|
|
34
49
|
"SIEError",
|
|
35
50
|
"ServerError",
|
|
51
|
+
"SpendLimitError",
|
|
36
52
|
]
|