lorebase-inference 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_inference/__init__.py +70 -0
- lorebase_inference/__main__.py +6 -0
- lorebase_inference/chat.py +447 -0
- lorebase_inference/cli.py +214 -0
- lorebase_inference/errors.py +54 -0
- lorebase_inference/fake.py +317 -0
- lorebase_inference/interfaces.py +151 -0
- lorebase_inference/local_embeddings.py +58 -0
- lorebase_inference/local_relations.py +157 -0
- lorebase_inference/local_spans.py +93 -0
- lorebase_inference/registry.py +313 -0
- lorebase_inference/remote_embeddings.py +100 -0
- lorebase_inference/settings.py +338 -0
- lorebase_inference/transport.py +233 -0
- lorebase_inference-0.2.3.dist-info/METADATA +13 -0
- lorebase_inference-0.2.3.dist-info/RECORD +18 -0
- lorebase_inference-0.2.3.dist-info/WHEEL +4 -0
- lorebase_inference-0.2.3.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""The `inference` settings section (design 16), registered with `lorebase_platform`.
|
|
2
|
+
|
|
3
|
+
Environment aliases override the YAML values of the default chat model:
|
|
4
|
+
`LLM_BASE_URL` (`inference.llm.base_url`), `LLM_MODEL` (`inference.llm.model`) and
|
|
5
|
+
`LLM_API_KEY` (`inference.llm.api_key`, optional, handled as a secret).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Literal, Self
|
|
12
|
+
|
|
13
|
+
from pydantic import Field, model_validator
|
|
14
|
+
|
|
15
|
+
from lorebase_platform.secrets import SecretValue
|
|
16
|
+
from lorebase_platform.settings import SectionModel, Settings, register_section
|
|
17
|
+
|
|
18
|
+
SECTION = "inference"
|
|
19
|
+
ENV_ALIASES = {
|
|
20
|
+
"LLM_BASE_URL": "llm.base_url",
|
|
21
|
+
"LLM_MODEL": "llm.model",
|
|
22
|
+
"LLM_API_KEY": "llm.api_key",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
ChatProvider = Literal["openai_compatible", "fake"]
|
|
26
|
+
EmbeddingProvider = Literal["fastembed", "openai_compatible", "fake"]
|
|
27
|
+
SpanProvider = Literal["gliner", "fake"]
|
|
28
|
+
RelationProvider = Literal["glirel", "cooccurrence", "fake"]
|
|
29
|
+
|
|
30
|
+
RESERVED_HEADERS = frozenset({"authorization", "content-type", "accept", "content-length", "host"})
|
|
31
|
+
"""Headers the client sets itself; `headers` cannot replace them."""
|
|
32
|
+
RESERVED_BODY_FIELDS = frozenset({"model", "messages", "stream", "response_format"})
|
|
33
|
+
"""Request fields the client sets itself; `extra_body` cannot replace them."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _check_headers(headers: dict[str, Any] | None) -> None:
|
|
37
|
+
for name in headers or {}:
|
|
38
|
+
if not name.strip() or name.strip().lower() in RESERVED_HEADERS:
|
|
39
|
+
raise ValueError(f"header {name!r} cannot be set in `headers`")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _check_extra_body(extra: dict[str, Any] | None) -> None:
|
|
43
|
+
for name in extra or {}:
|
|
44
|
+
if name in RESERVED_BODY_FIELDS:
|
|
45
|
+
raise ValueError(f"field {name!r} cannot be set in `extra_body`")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class RetrySettings(SectionModel):
|
|
49
|
+
"""Retries of HTTP calls on 429, 5xx and connection errors, with exponential backoff.
|
|
50
|
+
|
|
51
|
+
A `Retry-After` header is honored (capped at `backoff_max_s`); otherwise the wait before
|
|
52
|
+
retry `n` (from 0) is `min(backoff_max_s, backoff_base_s * 2**n)` with full jitter.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
max_retries: int = Field(default=3, ge=0, description="Retries after the first attempt.")
|
|
56
|
+
backoff_base_s: float = Field(default=0.5, gt=0)
|
|
57
|
+
backoff_max_s: float = Field(default=30, gt=0)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Price(SectionModel):
|
|
61
|
+
"""Price per million tokens in the operator's currency; zero for local models."""
|
|
62
|
+
|
|
63
|
+
input: float = Field(default=0, ge=0)
|
|
64
|
+
output: float = Field(default=0, ge=0)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class LlmTaskOverride(SectionModel):
|
|
68
|
+
"""Per-task override of the chat model; unset fields inherit from `llm`."""
|
|
69
|
+
|
|
70
|
+
provider: ChatProvider | None = None
|
|
71
|
+
base_url: str | None = None
|
|
72
|
+
model: str | None = None
|
|
73
|
+
api_key: SecretValue | None = None
|
|
74
|
+
structured_output: Literal["json_schema", "json_mode"] | None = None
|
|
75
|
+
json_schema_strict: bool | None = None
|
|
76
|
+
temperature: float | None = Field(default=None, ge=0)
|
|
77
|
+
max_concurrency: int | None = Field(default=None, ge=1)
|
|
78
|
+
timeout_s: float | None = Field(default=None, gt=0)
|
|
79
|
+
trust_env: bool | None = None
|
|
80
|
+
price_per_million_tokens: Price | None = None
|
|
81
|
+
headers: dict[str, SecretValue] | None = None
|
|
82
|
+
extra_body: dict[str, Any] | None = None
|
|
83
|
+
|
|
84
|
+
@model_validator(mode="after")
|
|
85
|
+
def _check_request_options(self) -> Self:
|
|
86
|
+
_check_headers(self.headers)
|
|
87
|
+
_check_extra_body(self.extra_body)
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class LlmSettings(SectionModel):
|
|
92
|
+
"""Chat model (tier 2). Any OpenAI-compatible endpoint, local or hosted."""
|
|
93
|
+
|
|
94
|
+
provider: ChatProvider = Field(
|
|
95
|
+
default="openai_compatible", description="openai_compatible or fake (tests)."
|
|
96
|
+
)
|
|
97
|
+
base_url: str = Field(
|
|
98
|
+
default="http://localhost:11434/v1",
|
|
99
|
+
min_length=1,
|
|
100
|
+
description="OpenAI-compatible endpoint. Default: local Ollama. Hosted providers and "
|
|
101
|
+
"a LiteLLM proxy are selected by changing this value (alias LLM_BASE_URL).",
|
|
102
|
+
)
|
|
103
|
+
model: str = Field(
|
|
104
|
+
default="qwen2.5:7b-instruct",
|
|
105
|
+
min_length=1,
|
|
106
|
+
description="Model name served by the endpoint (alias LLM_MODEL).",
|
|
107
|
+
)
|
|
108
|
+
api_key: SecretValue | None = Field(
|
|
109
|
+
default=None,
|
|
110
|
+
description="Literal or secret reference; omitted for local servers (alias LLM_API_KEY).",
|
|
111
|
+
)
|
|
112
|
+
structured_output: Literal["json_schema", "json_mode"] = Field(
|
|
113
|
+
default="json_schema",
|
|
114
|
+
description="How the endpoint is asked for schema-valid JSON. json_schema sends the "
|
|
115
|
+
"schema as response_format and steps down to json_mode when the server rejects it; "
|
|
116
|
+
"json_mode asks for a JSON object with the schema in the prompt. Answers are always "
|
|
117
|
+
"validated against the schema, with up to `repair_attempts` repair retries.",
|
|
118
|
+
)
|
|
119
|
+
json_schema_strict: bool | None = Field(
|
|
120
|
+
default=None,
|
|
121
|
+
description="Value of response_format.json_schema.strict in json_schema mode; null "
|
|
122
|
+
"leaves it out. Some endpoints require true and then refuse schemas outside their "
|
|
123
|
+
"strict subset, which steps the client down like any other rejection.",
|
|
124
|
+
)
|
|
125
|
+
repair_attempts: int = Field(
|
|
126
|
+
default=1, ge=0, le=3, description="Repair retries after an invalid answer."
|
|
127
|
+
)
|
|
128
|
+
temperature: float | None = Field(
|
|
129
|
+
default=0,
|
|
130
|
+
ge=0,
|
|
131
|
+
description="Sampling temperature; null leaves it out of the request, for models that "
|
|
132
|
+
"refuse the parameter.",
|
|
133
|
+
)
|
|
134
|
+
max_tokens: int | None = Field(
|
|
135
|
+
default=None, ge=1, description="Completion token cap; unset leaves it to the server."
|
|
136
|
+
)
|
|
137
|
+
max_concurrency: int = Field(default=2, ge=1, description="Parallel requests per process.")
|
|
138
|
+
timeout_s: float = Field(default=120, gt=0)
|
|
139
|
+
trust_env: bool = Field(
|
|
140
|
+
default=False,
|
|
141
|
+
description="Honor HTTP_PROXY, HTTPS_PROXY and NO_PROXY (and other httpx environment "
|
|
142
|
+
"settings) for this endpoint; false ignores them and connects directly.",
|
|
143
|
+
)
|
|
144
|
+
retry: RetrySettings = Field(default_factory=RetrySettings)
|
|
145
|
+
price_per_million_tokens: Price = Field(default_factory=Price)
|
|
146
|
+
headers: dict[str, SecretValue] = Field(
|
|
147
|
+
default_factory=dict,
|
|
148
|
+
description="Extra HTTP headers sent with every request, for example a workspace or "
|
|
149
|
+
"organization id a hosted provider requires. Values accept secret references.",
|
|
150
|
+
)
|
|
151
|
+
extra_body: dict[str, Any] = Field(
|
|
152
|
+
default_factory=dict,
|
|
153
|
+
description="Extra fields merged into every request body, for server-specific options "
|
|
154
|
+
'such as {"reasoning_effort": "none"}. model, messages, stream and response_format '
|
|
155
|
+
"are set by the client and cannot be overridden.",
|
|
156
|
+
)
|
|
157
|
+
tasks: dict[str, LlmTaskOverride] = Field(
|
|
158
|
+
default_factory=dict,
|
|
159
|
+
description="Optional overrides per task name, e.g. extraction, schema_discovery.",
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
@model_validator(mode="after")
|
|
163
|
+
def _check_request_options(self) -> Self:
|
|
164
|
+
_check_headers(self.headers)
|
|
165
|
+
_check_extra_body(self.extra_body)
|
|
166
|
+
return self
|
|
167
|
+
|
|
168
|
+
def for_task(self, task: str | None) -> Self:
|
|
169
|
+
"""Effective settings for a task: this model with the task's overrides applied.
|
|
170
|
+
|
|
171
|
+
A task that names another `base_url` uses only its own `api_key` and `headers`: the main
|
|
172
|
+
key and headers are never sent to an endpoint they were not configured for.
|
|
173
|
+
"""
|
|
174
|
+
override = self.tasks.get(task) if task else None
|
|
175
|
+
changes: dict[str, Any] = (
|
|
176
|
+
{
|
|
177
|
+
name: value
|
|
178
|
+
for name in type(override).model_fields
|
|
179
|
+
if (value := getattr(override, name)) is not None
|
|
180
|
+
}
|
|
181
|
+
if override
|
|
182
|
+
else {}
|
|
183
|
+
)
|
|
184
|
+
if override and override.temperature is None and "temperature" in override.model_fields_set:
|
|
185
|
+
changes["temperature"] = None # an explicit null leaves the parameter out
|
|
186
|
+
if override and override.base_url not in (None, self.base_url):
|
|
187
|
+
changes["api_key"] = override.api_key
|
|
188
|
+
changes["headers"] = override.headers or {}
|
|
189
|
+
return self.model_copy(update={**changes, "tasks": {}})
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class EmbeddingSettings(SectionModel):
|
|
193
|
+
"""Embedding model; its name and dimension define the vector index."""
|
|
194
|
+
|
|
195
|
+
provider: EmbeddingProvider = Field(
|
|
196
|
+
default="fastembed", description="fastembed (in-process), openai_compatible or fake."
|
|
197
|
+
)
|
|
198
|
+
model: str = Field(default="BAAI/bge-small-en-v1.5", min_length=1)
|
|
199
|
+
dims: int = Field(default=384, ge=1, description="Vector dimension produced by the model.")
|
|
200
|
+
base_url: str | None = Field(
|
|
201
|
+
default=None,
|
|
202
|
+
description="Endpoint for openai_compatible embeddings (Ollama or hosted); "
|
|
203
|
+
"for example http://localhost:11434/v1.",
|
|
204
|
+
)
|
|
205
|
+
api_key: SecretValue | None = None
|
|
206
|
+
batch_size: int = Field(default=32, ge=1, description="Texts per model call or request.")
|
|
207
|
+
query_prefix: str = Field(
|
|
208
|
+
default="",
|
|
209
|
+
description="Text put before a search query when it is embedded "
|
|
210
|
+
"(`lorebase-inference embed --query`); models such as E5 and BGE expect one, for "
|
|
211
|
+
'example "query: " or "Represent this sentence for searching relevant passages: ".',
|
|
212
|
+
)
|
|
213
|
+
document_prefix: str = Field(
|
|
214
|
+
default="",
|
|
215
|
+
description="Text put before every Record text Scribe embeds, for models that expect "
|
|
216
|
+
'one (E5: "passage: "). Part of the stored text hash and of the template fingerprint: '
|
|
217
|
+
"after changing it the next embedding pass embeds every text again.",
|
|
218
|
+
)
|
|
219
|
+
cache_dir: Path | None = Field(
|
|
220
|
+
default=None, description="fastembed: where model files are stored; unset: library default."
|
|
221
|
+
)
|
|
222
|
+
threads: int | None = Field(
|
|
223
|
+
default=None, ge=1, description="fastembed: CPU threads; unset: library default."
|
|
224
|
+
)
|
|
225
|
+
timeout_s: float = Field(default=60, gt=0, description="openai_compatible request timeout.")
|
|
226
|
+
trust_env: bool = Field(
|
|
227
|
+
default=False,
|
|
228
|
+
description="Honor HTTP_PROXY, HTTPS_PROXY and NO_PROXY (and other httpx environment "
|
|
229
|
+
"settings) for the openai_compatible endpoint; false ignores them and connects directly.",
|
|
230
|
+
)
|
|
231
|
+
retry: RetrySettings = Field(default_factory=RetrySettings)
|
|
232
|
+
|
|
233
|
+
@model_validator(mode="after")
|
|
234
|
+
def _endpoint_required(self) -> Self:
|
|
235
|
+
if self.provider == "openai_compatible" and not self.base_url:
|
|
236
|
+
raise ValueError("base_url is required when provider is openai_compatible")
|
|
237
|
+
return self
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class ExtractorSettings(SectionModel):
|
|
241
|
+
"""One tier-1 local extractor."""
|
|
242
|
+
|
|
243
|
+
enabled: bool = True
|
|
244
|
+
model: str = Field(min_length=1, description="Checkpoint name; see the weight license note.")
|
|
245
|
+
threshold: float = Field(default=0.5, ge=0, le=1)
|
|
246
|
+
cache_dir: Path | None = Field(
|
|
247
|
+
default=None, description="Where checkpoints are stored; unset: library default."
|
|
248
|
+
)
|
|
249
|
+
window_chars: int = Field(
|
|
250
|
+
default=1500,
|
|
251
|
+
ge=100,
|
|
252
|
+
description="Longer texts are cut at whitespace into windows of at most this size, "
|
|
253
|
+
"because the models read a few hundred tokens at a time.",
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class SpacySettings(SectionModel):
|
|
258
|
+
"""Optional spaCy pipeline."""
|
|
259
|
+
|
|
260
|
+
enabled: bool = False
|
|
261
|
+
pipeline: str = Field(default="en_core_web_sm", min_length=1)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class CooccurrenceSettings(SectionModel):
|
|
265
|
+
"""Limits of the deterministic relation fallback (`relation_provider: cooccurrence`)."""
|
|
266
|
+
|
|
267
|
+
max_pairs_per_sentence: int = Field(
|
|
268
|
+
default=200, ge=1, description="Candidate pairs kept per sentence, nearest first."
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class LocalExtractorSettings(SectionModel):
|
|
273
|
+
"""Tier-1 extractors; check each checkpoint's weight license before enabling."""
|
|
274
|
+
|
|
275
|
+
provider: SpanProvider = Field(default="gliner", description="gliner or fake (tests).")
|
|
276
|
+
relation_provider: RelationProvider = Field(
|
|
277
|
+
default="glirel",
|
|
278
|
+
description="glirel, or cooccurrence (deterministic pairing of spans that share a "
|
|
279
|
+
"sentence, no model), or fake (same as cooccurrence, for tests; the default when "
|
|
280
|
+
"`provider` is fake).",
|
|
281
|
+
)
|
|
282
|
+
gliner: ExtractorSettings = ExtractorSettings(model="urchade/gliner_multi-v2.1")
|
|
283
|
+
glirel: ExtractorSettings = ExtractorSettings(model="jackboyla/glirel-large-v0")
|
|
284
|
+
cooccurrence: CooccurrenceSettings = Field(default_factory=CooccurrenceSettings)
|
|
285
|
+
spacy: SpacySettings = Field(default_factory=SpacySettings)
|
|
286
|
+
|
|
287
|
+
@model_validator(mode="before")
|
|
288
|
+
@classmethod
|
|
289
|
+
def _fake_provider_implies_fake_relations(cls, data: Any) -> Any:
|
|
290
|
+
"""`provider: fake` selects the offline relation provider too, unless one is named."""
|
|
291
|
+
if isinstance(data, dict) and data.get("provider") == "fake":
|
|
292
|
+
return {"relation_provider": "fake", **data}
|
|
293
|
+
return data
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class FakeSettings(SectionModel):
|
|
297
|
+
"""Behavior of `provider: fake`, used by tests and offline demos."""
|
|
298
|
+
|
|
299
|
+
fixtures_dir: Path | None = Field(
|
|
300
|
+
default=None,
|
|
301
|
+
description="Directory of `<prompt-hash>.json` files holding canned chat answers.",
|
|
302
|
+
)
|
|
303
|
+
dictionary: dict[str, list[str]] = Field(
|
|
304
|
+
default_factory=dict,
|
|
305
|
+
description="Label name -> surface forms matched by the fake span extractor.",
|
|
306
|
+
)
|
|
307
|
+
max_schema_depth: int = Field(
|
|
308
|
+
default=6, ge=1, description="Nesting depth beyond which optional properties are omitted."
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class InferenceSettings(SectionModel):
|
|
313
|
+
"""All model-related configuration. Used by Scribe only."""
|
|
314
|
+
|
|
315
|
+
llm: LlmSettings = Field(default_factory=LlmSettings)
|
|
316
|
+
embeddings: EmbeddingSettings = Field(default_factory=EmbeddingSettings)
|
|
317
|
+
local_extractors: LocalExtractorSettings = Field(default_factory=LocalExtractorSettings)
|
|
318
|
+
fake: FakeSettings = Field(default_factory=FakeSettings)
|
|
319
|
+
mode_memory_key: SecretValue | None = Field(
|
|
320
|
+
default=None,
|
|
321
|
+
description="Secret that keys the digests under which the structured output mode of "
|
|
322
|
+
"each chat endpoint is remembered across restarts, e.g. secret://env/LOREBASE_MODE_KEY; "
|
|
323
|
+
"without it the digests are keyed by a fixed label only. Changing it forgets the modes.",
|
|
324
|
+
)
|
|
325
|
+
mode_memory_ttl_s: float = Field(
|
|
326
|
+
default=7 * 24 * 3600,
|
|
327
|
+
gt=0,
|
|
328
|
+
description="Seconds a remembered structured output mode counts; after that the "
|
|
329
|
+
"endpoint is tried with the configured mode again.",
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
register_section(SECTION, InferenceSettings, env_aliases=ENV_ALIASES)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def inference_settings(settings: Settings) -> InferenceSettings:
|
|
337
|
+
"""The validated `inference` section of loaded settings."""
|
|
338
|
+
return settings.section(SECTION, InferenceSettings)
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""HTTP client shared by the endpoint providers: timeouts, retries and backoff.
|
|
2
|
+
|
|
3
|
+
Speaks JSON over plain HTTP with `httpx`; no vendor library is involved. Calls are retried on
|
|
4
|
+
429, 500, 502, 503, 504 and on connection errors or timeouts, up to `retry.max_retries` times.
|
|
5
|
+
A `Retry-After` header (seconds or HTTP date) sets the wait, capped at `retry.backoff_max_s`;
|
|
6
|
+
otherwise the wait is exponential backoff with full jitter. Request and response bodies are
|
|
7
|
+
never logged: they hold prompts and documents.
|
|
8
|
+
|
|
9
|
+
Error text sent by the server is kept in `EndpointError` only after anything that looks like a
|
|
10
|
+
credential is removed, and not at all for 401 and 403, where providers often echo the key.
|
|
11
|
+
Proxy variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`) are ignored unless `trust_env` is set.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
import logging
|
|
18
|
+
import math
|
|
19
|
+
import random
|
|
20
|
+
import re
|
|
21
|
+
import time
|
|
22
|
+
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
from email.utils import parsedate_to_datetime
|
|
25
|
+
from types import TracebackType
|
|
26
|
+
from typing import Any, Self
|
|
27
|
+
|
|
28
|
+
import httpx
|
|
29
|
+
from pydantic import SecretStr
|
|
30
|
+
|
|
31
|
+
from lorebase_inference.errors import EndpointError
|
|
32
|
+
from lorebase_inference.settings import RetrySettings
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger("lorebase.inference")
|
|
35
|
+
|
|
36
|
+
RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
|
|
37
|
+
_DETAIL_LIMIT = 300
|
|
38
|
+
_SCRUB_WINDOW = _DETAIL_LIMIT * 4
|
|
39
|
+
CREDENTIAL_STATUS = frozenset({401, 403})
|
|
40
|
+
REDACTED = "[redacted]"
|
|
41
|
+
_MIN_SECRET_CHARS = 8
|
|
42
|
+
_CREDENTIAL = re.compile(
|
|
43
|
+
r"\bbearer\s+[^\s\"',;]+" # an echoed Authorization value
|
|
44
|
+
r"|(?<![A-Za-z0-9])(?:sk|pk|rk)[-_][A-Za-z0-9_\-*.]{6,}" # sk-..., sk_live_..., masked too
|
|
45
|
+
# long base64, base64url or hex runs holding both letters and digits
|
|
46
|
+
r"|(?<![A-Za-z0-9+/_\-])(?=[A-Za-z0-9+/_\-]*\d)(?=[A-Za-z0-9+/_\-]*[A-Za-z])"
|
|
47
|
+
r"[A-Za-z0-9+/_\-]{32,}={0,2}",
|
|
48
|
+
re.IGNORECASE,
|
|
49
|
+
)
|
|
50
|
+
_CONNECT_TIMEOUT_S = 10.0
|
|
51
|
+
|
|
52
|
+
Sleep = Callable[[float], Awaitable[None]]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_retry_after(
|
|
56
|
+
value: str | None, *, now: Callable[[], datetime] | None = None
|
|
57
|
+
) -> float | None:
|
|
58
|
+
"""Seconds to wait according to a `Retry-After` header value, or `None` when unusable."""
|
|
59
|
+
if not value:
|
|
60
|
+
return None
|
|
61
|
+
value = value.strip()
|
|
62
|
+
try:
|
|
63
|
+
seconds = float(value)
|
|
64
|
+
except ValueError:
|
|
65
|
+
try:
|
|
66
|
+
moment = parsedate_to_datetime(value)
|
|
67
|
+
except (TypeError, ValueError):
|
|
68
|
+
return None
|
|
69
|
+
if moment.tzinfo is None:
|
|
70
|
+
moment = moment.replace(tzinfo=UTC)
|
|
71
|
+
seconds = float((moment - (now or (lambda: datetime.now(UTC)))()).total_seconds())
|
|
72
|
+
return None if math.isnan(seconds) else max(seconds, 0.0)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def scrub_credentials(text: str, *, secrets: Iterable[str] = ()) -> str:
|
|
76
|
+
"""`text` with the given secrets and anything shaped like a credential replaced."""
|
|
77
|
+
for secret in secrets:
|
|
78
|
+
if len(secret) >= _MIN_SECRET_CHARS:
|
|
79
|
+
text = text.replace(secret, REDACTED)
|
|
80
|
+
return _CREDENTIAL.sub(REDACTED, text)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def error_detail(response: httpx.Response, *, secrets: Iterable[str] = ()) -> str:
|
|
84
|
+
"""Error text of a failed response: the API error message when present, else the body.
|
|
85
|
+
|
|
86
|
+
Whitespace is collapsed, credentials are scrubbed (see `scrub_credentials`) and the result
|
|
87
|
+
is cut to a bounded excerpt.
|
|
88
|
+
"""
|
|
89
|
+
try:
|
|
90
|
+
body = response.json()
|
|
91
|
+
except ValueError:
|
|
92
|
+
text = response.text
|
|
93
|
+
else:
|
|
94
|
+
error = body.get("error") if isinstance(body, dict) else None
|
|
95
|
+
if isinstance(error, dict):
|
|
96
|
+
text = str(error.get("message") or error)
|
|
97
|
+
elif error is not None:
|
|
98
|
+
text = str(error)
|
|
99
|
+
else:
|
|
100
|
+
text = str(body)
|
|
101
|
+
excerpt = " ".join(text[: _SCRUB_WINDOW * 4].split())[:_SCRUB_WINDOW]
|
|
102
|
+
return scrub_credentials(excerpt, secrets=secrets)[:_DETAIL_LIMIT]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class EndpointClient:
|
|
106
|
+
"""JSON POST client for one HTTP endpoint with bounded concurrency and retries."""
|
|
107
|
+
|
|
108
|
+
def __init__(
|
|
109
|
+
self,
|
|
110
|
+
*,
|
|
111
|
+
base_url: str,
|
|
112
|
+
api_key: SecretStr | None,
|
|
113
|
+
timeout_s: float,
|
|
114
|
+
retry: RetrySettings,
|
|
115
|
+
max_concurrency: int = 4,
|
|
116
|
+
trust_env: bool = False,
|
|
117
|
+
headers: Mapping[str, SecretStr] | None = None,
|
|
118
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
119
|
+
sleep: Sleep | None = None,
|
|
120
|
+
jitter: Callable[[], float] = random.random,
|
|
121
|
+
) -> None:
|
|
122
|
+
if not base_url.startswith(("http://", "https://")):
|
|
123
|
+
raise ValueError(f"base_url must start with http:// or https://, got {base_url!r}")
|
|
124
|
+
self._base_url = base_url.rstrip("/")
|
|
125
|
+
self._key = api_key
|
|
126
|
+
self._headers = dict(headers or {})
|
|
127
|
+
self._timeout = httpx.Timeout(timeout_s, connect=min(_CONNECT_TIMEOUT_S, timeout_s))
|
|
128
|
+
self._retry = retry
|
|
129
|
+
self._slots = asyncio.Semaphore(max_concurrency)
|
|
130
|
+
self._trust_env = trust_env
|
|
131
|
+
self._transport = transport
|
|
132
|
+
self._sleep: Sleep = sleep or asyncio.sleep
|
|
133
|
+
self._jitter = jitter
|
|
134
|
+
self._client: httpx.AsyncClient | None = None
|
|
135
|
+
|
|
136
|
+
async def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
137
|
+
"""POST `payload` as JSON to `path` under the base URL and return the JSON object."""
|
|
138
|
+
client = self._http()
|
|
139
|
+
headers = {name: value.get_secret_value() for name, value in self._headers.items()}
|
|
140
|
+
headers.update({"Content-Type": "application/json", "Accept": "application/json"})
|
|
141
|
+
if self._key is not None and self._key.get_secret_value():
|
|
142
|
+
headers["Authorization"] = f"Bearer {self._key.get_secret_value()}"
|
|
143
|
+
async with self._slots:
|
|
144
|
+
attempt = 0
|
|
145
|
+
while True:
|
|
146
|
+
started = time.monotonic()
|
|
147
|
+
retry_after: float | None = None
|
|
148
|
+
try:
|
|
149
|
+
response = await client.post(
|
|
150
|
+
self._base_url + path, json=payload, headers=headers
|
|
151
|
+
)
|
|
152
|
+
except httpx.TransportError as exc:
|
|
153
|
+
reason = type(exc).__name__
|
|
154
|
+
failure = EndpointError(f"{path}: request failed ({reason})")
|
|
155
|
+
else:
|
|
156
|
+
logger.debug(
|
|
157
|
+
"POST %s -> %d in %.0f ms (attempt %d)",
|
|
158
|
+
path,
|
|
159
|
+
response.status_code,
|
|
160
|
+
(time.monotonic() - started) * 1000,
|
|
161
|
+
attempt + 1,
|
|
162
|
+
)
|
|
163
|
+
if response.status_code < 400:
|
|
164
|
+
return _json_object(path, response)
|
|
165
|
+
reason = f"HTTP {response.status_code}"
|
|
166
|
+
failure = self._failure(path, response)
|
|
167
|
+
if response.status_code not in RETRYABLE_STATUS:
|
|
168
|
+
raise failure
|
|
169
|
+
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
170
|
+
if attempt >= self._retry.max_retries:
|
|
171
|
+
raise failure
|
|
172
|
+
delay = self._delay(attempt, retry_after)
|
|
173
|
+
logger.warning(
|
|
174
|
+
"%s failed (%s); retry %d/%d in %.1f s",
|
|
175
|
+
path,
|
|
176
|
+
reason,
|
|
177
|
+
attempt + 1,
|
|
178
|
+
self._retry.max_retries,
|
|
179
|
+
delay,
|
|
180
|
+
)
|
|
181
|
+
await self._sleep(delay)
|
|
182
|
+
attempt += 1
|
|
183
|
+
|
|
184
|
+
def _failure(self, path: str, response: httpx.Response) -> EndpointError:
|
|
185
|
+
status = response.status_code
|
|
186
|
+
if status in CREDENTIAL_STATUS:
|
|
187
|
+
return EndpointError(
|
|
188
|
+
f"{path}: HTTP {status}: credentials refused (server text omitted)", status=status
|
|
189
|
+
)
|
|
190
|
+
secrets = [self._key.get_secret_value()] if self._key is not None else []
|
|
191
|
+
secrets.extend(value.get_secret_value() for value in self._headers.values())
|
|
192
|
+
detail = error_detail(response, secrets=secrets)
|
|
193
|
+
return EndpointError(f"{path}: HTTP {status}: {detail}", status=status, detail=detail)
|
|
194
|
+
|
|
195
|
+
def _delay(self, attempt: int, retry_after: float | None) -> float:
|
|
196
|
+
cap = self._retry.backoff_max_s
|
|
197
|
+
if retry_after is not None:
|
|
198
|
+
return min(retry_after, cap)
|
|
199
|
+
return float(self._jitter() * min(cap, self._retry.backoff_base_s * 2**attempt))
|
|
200
|
+
|
|
201
|
+
def _http(self) -> httpx.AsyncClient:
|
|
202
|
+
if self._client is None:
|
|
203
|
+
self._client = httpx.AsyncClient(
|
|
204
|
+
timeout=self._timeout, transport=self._transport, trust_env=self._trust_env
|
|
205
|
+
)
|
|
206
|
+
return self._client
|
|
207
|
+
|
|
208
|
+
async def aclose(self) -> None:
|
|
209
|
+
"""Release the connection pool."""
|
|
210
|
+
if self._client is not None:
|
|
211
|
+
await self._client.aclose()
|
|
212
|
+
self._client = None
|
|
213
|
+
|
|
214
|
+
async def __aenter__(self) -> Self:
|
|
215
|
+
return self
|
|
216
|
+
|
|
217
|
+
async def __aexit__(
|
|
218
|
+
self,
|
|
219
|
+
exc_type: type[BaseException] | None,
|
|
220
|
+
exc: BaseException | None,
|
|
221
|
+
tb: TracebackType | None,
|
|
222
|
+
) -> None:
|
|
223
|
+
await self.aclose()
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _json_object(path: str, response: httpx.Response) -> dict[str, Any]:
|
|
227
|
+
try:
|
|
228
|
+
body = response.json()
|
|
229
|
+
except ValueError:
|
|
230
|
+
raise EndpointError(f"{path}: response is not JSON", status=response.status_code) from None
|
|
231
|
+
if not isinstance(body, dict):
|
|
232
|
+
raise EndpointError(f"{path}: response is not a JSON object", status=response.status_code)
|
|
233
|
+
return body
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lorebase-inference
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: Lorebase inference: model-agnostic chat, embedding and span extraction providers
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Requires-Dist: httpx>=0.27
|
|
7
|
+
Requires-Dist: jsonschema>=4.22
|
|
8
|
+
Requires-Dist: lorebase-platform<0.3,>=0.2
|
|
9
|
+
Provides-Extra: fastembed
|
|
10
|
+
Requires-Dist: fastembed>=0.4; extra == 'fastembed'
|
|
11
|
+
Provides-Extra: local-extractors
|
|
12
|
+
Requires-Dist: gliner>=0.2; extra == 'local-extractors'
|
|
13
|
+
Requires-Dist: glirel>=1.0; extra == 'local-extractors'
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
lorebase_inference/__init__.py,sha256=p_yoFoAkIwPhTGPvLs8X3xicon53-M3mAmd9ln3cClE,1599
|
|
2
|
+
lorebase_inference/__main__.py,sha256=AO0XYw-Tv6GbCrSj9hNnnlUhwp7YqmIryub8v30ikDA,146
|
|
3
|
+
lorebase_inference/chat.py,sha256=UttGW97BalG1gJ1biEK7ngC3we5ww3nlQBcTubVzV8A,17312
|
|
4
|
+
lorebase_inference/cli.py,sha256=sk7IZzkEpq1PzusVSbmTcRNkTZpxXUtUZwAIwLJkjLI,8512
|
|
5
|
+
lorebase_inference/errors.py,sha256=MH4Rmc4wYSFdDOipWmqsFIkN2FjtGnkVOXjknCjZRLA,1828
|
|
6
|
+
lorebase_inference/fake.py,sha256=O1I_w8XYyNwBUJHSwPe0eZKcQ8qyTYBMg652q9RdQ1g,12789
|
|
7
|
+
lorebase_inference/interfaces.py,sha256=bpTxcFNN-Nf79ZcpNXa7mNg_GRn2Ey35V4a3OOsfCnM,4333
|
|
8
|
+
lorebase_inference/local_embeddings.py,sha256=gtZ3cCzOuWZhW2p1xMyMEaBCoveCP8h5zAHIqxslIhM,2352
|
|
9
|
+
lorebase_inference/local_relations.py,sha256=P_hiMBROZVHt264SA2y80PSF0Gl-kQSBFsSu2zLLqRs,7465
|
|
10
|
+
lorebase_inference/local_spans.py,sha256=OkDiCMY0LDE3qT9AtaCMfX3y49ZOVN97My1Za0nBGyM,4035
|
|
11
|
+
lorebase_inference/registry.py,sha256=hx-Jt47WHsQOf_q7TuNkVEBYdKMHdyHhimm1j7QlrqU,12340
|
|
12
|
+
lorebase_inference/remote_embeddings.py,sha256=1_3v6txuKarJuGcuGwHbci-UhYlykyAjOBkRrmNGDNU,3936
|
|
13
|
+
lorebase_inference/settings.py,sha256=dsSjItAgRCbRGB3DzW5Zx3dcovZf49dNadqjVA5PPdc,14507
|
|
14
|
+
lorebase_inference/transport.py,sha256=89yOjzf4jnCuy4l_T4GDvj-7vdpT1jsY7tMi2dIzDqo,9331
|
|
15
|
+
lorebase_inference-0.2.3.dist-info/METADATA,sha256=_qlKWaDQ1hgWtSTzDt4aHxfgsVwjdXHiBm63VF5r0VM,501
|
|
16
|
+
lorebase_inference-0.2.3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
17
|
+
lorebase_inference-0.2.3.dist-info/entry_points.txt,sha256=tNQ5m5Coi1HOA-C6FV7194IRZkPy5xumhjIJrrfG-3A,67
|
|
18
|
+
lorebase_inference-0.2.3.dist-info/RECORD,,
|