python-corekit 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/api/application.py +47 -9
- corekit/api/lifespan.py +26 -3
- corekit/concurrency/__init__.py +2 -2
- corekit/concurrency/decorators.py +32 -5
- corekit/concurrency/thread_local.py +2 -2
- corekit/concurrency/worker.py +9 -0
- corekit/config/loader.py +42 -5
- corekit/config/settings.py +11 -1
- corekit/connections/__init__.py +7 -1
- corekit/connections/connectable.py +45 -4
- corekit/connections/redis/connection.py +53 -10
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +39 -5
- corekit/connections/sql/fields/__init__.py +2 -2
- corekit/connections/sql/fields/jsonb.py +13 -6
- corekit/connections/sql/migration/__init__.py +4 -0
- corekit/connections/sql/migration/operations.py +69 -2
- corekit/connections/sql/operations/base.py +11 -2
- corekit/connections/sql/operations/statements.py +25 -5
- corekit/connections/sql/table.py +7 -29
- corekit/crypto/__init__.py +3 -1
- corekit/crypto/constants.py +2 -2
- corekit/crypto/hasher.py +9 -4
- corekit/data/dataset.py +8 -2
- corekit/data/expressions/__init__.py +3 -3
- corekit/data/expressions/comparison.py +19 -80
- corekit/data/expressions/expression.py +0 -32
- corekit/data/expressions/operator.py +13 -28
- corekit/data/stats.py +3 -0
- corekit/decorators/exception_handling.py +36 -8
- corekit/docker/watchdog.py +50 -31
- corekit/etl/__init__.py +2 -1
- corekit/etl/connection.py +14 -12
- corekit/etl/extract/extractor.py +6 -13
- corekit/etl/orchestrator.py +19 -2
- corekit/etl/schemas.py +2 -2
- corekit/etl/transform/transformer.py +4 -1
- corekit/events/publisher.py +1 -1
- corekit/events/reader.py +26 -21
- corekit/events/sse.py +4 -1
- corekit/events/websocket.py +24 -11
- corekit/exceptions/__init__.py +24 -9
- corekit/exceptions/base.py +139 -10
- corekit/exceptions/enum.py +17 -0
- corekit/exceptions/types.py +6 -6
- corekit/files/__init__.py +2 -4
- corekit/files/base.py +15 -2
- corekit/files/enum.py +0 -5
- corekit/files/json.py +16 -2
- corekit/http/__init__.py +48 -5
- corekit/http/api.py +24 -0
- corekit/http/client.py +133 -75
- corekit/http/exceptions.py +140 -0
- corekit/http/response.py +50 -1
- corekit/http/status.py +89 -0
- corekit/http/stream.py +110 -0
- corekit/jobs/runner.py +12 -1
- corekit/jobs/task.py +23 -2
- corekit/llm/__init__.py +134 -0
- corekit/llm/client.py +179 -0
- corekit/llm/enum.py +123 -0
- corekit/llm/events.py +96 -0
- corekit/llm/messages.py +173 -0
- corekit/llm/prompts/__init__.py +19 -0
- corekit/llm/prompts/enum.py +54 -0
- corekit/llm/prompts/exceptions.py +22 -0
- corekit/llm/prompts/loader.py +139 -0
- corekit/llm/prompts/template.py +53 -0
- corekit/llm/protocols.py +65 -0
- corekit/llm/streaming.py +149 -0
- corekit/llm/tools/__init__.py +19 -0
- corekit/llm/tools/base.py +118 -0
- corekit/llm/tools/detection.py +99 -0
- corekit/llm/tools/loop.py +255 -0
- corekit/llm/tools/registry.py +103 -0
- corekit/llm/wire.py +199 -0
- corekit/log_monitor/models.py +8 -2
- corekit/log_monitor/service.py +77 -38
- corekit/notifications/base.py +18 -10
- corekit/observability/__init__.py +12 -5
- corekit/observability/benchmarkable.py +37 -5
- corekit/observability/loggable.py +21 -0
- corekit/observability/request_context.py +55 -2
- corekit/observability/timing/split.py +14 -0
- corekit/observability/timing/timer.py +33 -9
- corekit/registry/__init__.py +2 -2
- corekit/registry/registry.py +55 -14
- corekit/schemas/__init__.py +2 -1
- corekit/schemas/enum.py +22 -1
- corekit/schemas/types.py +6 -1
- corekit/schemas/version.py +58 -0
- corekit/serialization/__init__.py +2 -0
- corekit/serialization/pickle_file.py +61 -0
- corekit/serialization/serializable.py +22 -2
- corekit/serialization/serializer.py +9 -2
- corekit/utils/__init__.py +2 -1
- corekit/utils/collections.py +38 -14
- corekit/utils/payload.py +12 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/METADATA +38 -9
- python_corekit-0.4.0.dist-info/RECORD +165 -0
- corekit/constants.py +0 -45
- corekit/exceptions/http/exceptions.py +0 -37
- corekit/files/pickle.py +0 -12
- python_corekit-0.2.0.dist-info/RECORD +0 -143
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/WHEEL +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/top_level.txt +0 -0
corekit/http/client.py
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"""
|
|
2
2
|
A small HTTP client with retries.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
``BaseHttpClient`` is the transport: relative paths, optional strict URL
|
|
5
|
+
checking, retries for statuses the typed HTTP exceptions mark as
|
|
6
|
+
retryable, and ``stream_request`` for SSE / chunked bodies. Subclass
|
|
7
|
+
``BaseApiClient`` when the client always talks to one API and a foreign
|
|
8
|
+
absolute URL should be a bug.
|
|
5
9
|
|
|
6
10
|
class GithubClient(BaseApiClient):
|
|
7
11
|
'''
|
|
@@ -17,46 +21,81 @@ Subclass and give it a base URL::
|
|
|
17
21
|
return {"Authorization": f"Bearer {self.token}"}
|
|
18
22
|
|
|
19
23
|
client = GithubClient()
|
|
20
|
-
response = client.get("/user") # or await client.
|
|
24
|
+
response = client.get("/user") # or await client.get("/user")
|
|
21
25
|
response.data["login"]
|
|
22
26
|
|
|
23
27
|
Every response comes back as a ``BaseApiResponse``, so callers see one shape
|
|
24
|
-
regardless of what the endpoint returned.
|
|
25
|
-
status are retried with exponential backoff.
|
|
28
|
+
regardless of what the endpoint returned.
|
|
26
29
|
"""
|
|
27
30
|
|
|
31
|
+
from collections.abc import AsyncIterator
|
|
32
|
+
from contextlib import asynccontextmanager
|
|
28
33
|
from http import HTTPMethod
|
|
29
34
|
from typing import Any
|
|
30
|
-
from urllib.parse import urljoin
|
|
35
|
+
from urllib.parse import urljoin, urlsplit, urlunsplit
|
|
31
36
|
|
|
32
37
|
import httpx
|
|
33
38
|
|
|
39
|
+
from corekit.concurrency.decorators import allow_sync
|
|
40
|
+
from corekit.http.exceptions import RETRYABLE_STATUS_CODES, UnsupportedMethodError, URLMismatchError
|
|
34
41
|
from corekit.http.exponential_backoff import ExponentialBackoff
|
|
35
42
|
from corekit.http.response import BaseApiResponse
|
|
43
|
+
from corekit.http.status import HTTPStatusCode
|
|
36
44
|
from corekit.observability.benchmarkable import Benchmarkable
|
|
37
45
|
|
|
38
|
-
__all__ = ["
|
|
46
|
+
__all__ = ["BaseHttpClient"]
|
|
39
47
|
|
|
40
|
-
# Statuses worth retrying: the server is busy or briefly unavailable, so the
|
|
41
|
-
# same request may well succeed shortly. A 4xx other than 429 will not.
|
|
42
|
-
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})
|
|
43
48
|
|
|
44
|
-
|
|
45
|
-
class URLMismatchError(Exception):
|
|
49
|
+
def _without_query(url: str) -> str:
|
|
46
50
|
"""
|
|
47
|
-
|
|
51
|
+
Drop the query string before a retry warning is logged.
|
|
48
52
|
"""
|
|
53
|
+
parts = urlsplit(url)
|
|
54
|
+
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_DEFAULT_METHODS = frozenset(
|
|
58
|
+
{
|
|
59
|
+
HTTPMethod.GET,
|
|
60
|
+
HTTPMethod.POST,
|
|
61
|
+
HTTPMethod.PUT,
|
|
62
|
+
HTTPMethod.PATCH,
|
|
63
|
+
HTTPMethod.DELETE,
|
|
64
|
+
}
|
|
65
|
+
)
|
|
49
66
|
|
|
50
67
|
|
|
51
|
-
class
|
|
68
|
+
class BaseHttpClient(Benchmarkable):
|
|
52
69
|
"""
|
|
53
|
-
Base for
|
|
70
|
+
Base for HTTP clients. Override ``base_url`` and, usually, ``headers``.
|
|
71
|
+
|
|
72
|
+
``strict=False`` accepts a foreign absolute URL as-is. ``BaseApiClient``
|
|
73
|
+
turns that into an error.
|
|
54
74
|
"""
|
|
55
75
|
|
|
56
|
-
def __init__(
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
retries: int = 3,
|
|
79
|
+
timeout: float = 30.0,
|
|
80
|
+
strict: bool = False,
|
|
81
|
+
base_delay: float = 1,
|
|
82
|
+
http_client: httpx.AsyncClient | None = None,
|
|
83
|
+
) -> None:
|
|
57
84
|
super().__init__()
|
|
58
85
|
self.retries = retries
|
|
59
86
|
self.timeout = timeout
|
|
87
|
+
self.strict = strict
|
|
88
|
+
self.base_delay = base_delay
|
|
89
|
+
# Optional pool. Left unset by default: @allow_sync runs sync calls on a
|
|
90
|
+
# throwaway loop, and an AsyncClient cannot move between loops.
|
|
91
|
+
self._http_client = http_client
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def supported_methods(self) -> frozenset[HTTPMethod]:
|
|
95
|
+
"""
|
|
96
|
+
HTTP methods this client will send. Override to narrow or extend the set.
|
|
97
|
+
"""
|
|
98
|
+
return _DEFAULT_METHODS
|
|
60
99
|
|
|
61
100
|
@property
|
|
62
101
|
def headers(self) -> dict[str, str]:
|
|
@@ -77,16 +116,18 @@ class BaseApiClient(Benchmarkable):
|
|
|
77
116
|
Resolve a path against the base URL, accepting an absolute URL that
|
|
78
117
|
already matches it.
|
|
79
118
|
"""
|
|
80
|
-
if url.startswith(self.base_url):
|
|
119
|
+
if self.base_url and url.startswith(self.base_url):
|
|
81
120
|
return url
|
|
82
121
|
if url.startswith("http"):
|
|
83
|
-
|
|
122
|
+
if self.strict:
|
|
123
|
+
raise URLMismatchError(url, self.base_url)
|
|
124
|
+
return url
|
|
84
125
|
return urljoin(self.base_url, url)
|
|
85
126
|
|
|
86
127
|
@staticmethod
|
|
87
|
-
def _to_response(raw:
|
|
128
|
+
def _to_response(raw: httpx.Response) -> BaseApiResponse:
|
|
88
129
|
"""
|
|
89
|
-
Normalize
|
|
130
|
+
Normalize a httpx response into a BaseApiResponse.
|
|
90
131
|
|
|
91
132
|
The body is decoded as JSON when it parses, and left as text otherwise,
|
|
92
133
|
so a caller never has to guard against a non-JSON error page.
|
|
@@ -99,78 +140,95 @@ class BaseApiClient(Benchmarkable):
|
|
|
99
140
|
data = {"data": data}
|
|
100
141
|
|
|
101
142
|
return BaseApiResponse(
|
|
102
|
-
status_code=raw.status_code,
|
|
143
|
+
status_code=HTTPStatusCode.coerce(raw.status_code),
|
|
103
144
|
text=raw.text,
|
|
104
145
|
data=data,
|
|
105
146
|
headers=dict(raw.headers),
|
|
106
147
|
cookies=dict(raw.cookies),
|
|
107
148
|
)
|
|
108
149
|
|
|
109
|
-
def
|
|
110
|
-
return ExponentialBackoff(retries=self.retries)
|
|
111
|
-
|
|
112
|
-
def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
113
|
-
"""
|
|
114
|
-
Send a request, retrying retryable statuses.
|
|
115
|
-
"""
|
|
116
|
-
|
|
117
|
-
target = self._build_url(url)
|
|
118
|
-
backoff = self._new_backoff()
|
|
119
|
-
|
|
120
|
-
while True:
|
|
121
|
-
with httpx.Client(timeout=self.timeout) as client:
|
|
122
|
-
raw = client.request(str(method), target, headers=self.headers, **kwargs)
|
|
123
|
-
|
|
124
|
-
if raw.status_code not in RETRYABLE_STATUS_CODES or backoff.did_timeout():
|
|
125
|
-
return self._to_response(raw)
|
|
126
|
-
|
|
127
|
-
self.warning(f"{method} {target} returned {raw.status_code}; retrying")
|
|
128
|
-
backoff.wait()
|
|
129
|
-
|
|
130
|
-
async def async_request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
150
|
+
async def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
131
151
|
"""
|
|
132
152
|
Send a request asynchronously, retrying retryable statuses.
|
|
133
153
|
"""
|
|
154
|
+
if method not in self.supported_methods:
|
|
155
|
+
raise UnsupportedMethodError(method)
|
|
134
156
|
|
|
135
157
|
target = self._build_url(url)
|
|
136
|
-
|
|
137
|
-
|
|
158
|
+
headers = dict(self.headers)
|
|
159
|
+
extra_headers = kwargs.pop("headers", None)
|
|
160
|
+
if extra_headers:
|
|
161
|
+
headers.update(extra_headers)
|
|
162
|
+
# Fresh each request: ExponentialBackoff keeps its own attempt count.
|
|
163
|
+
backoff = ExponentialBackoff(retries=self.retries, base_delay=self.base_delay)
|
|
164
|
+
|
|
165
|
+
if self._http_client is not None:
|
|
166
|
+
return await self._exchange(self._http_client, method, target, headers, backoff, kwargs)
|
|
167
|
+
|
|
168
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
169
|
+
return await self._exchange(client, method, target, headers, backoff, kwargs)
|
|
170
|
+
|
|
171
|
+
async def _exchange(
|
|
172
|
+
self,
|
|
173
|
+
client: httpx.AsyncClient,
|
|
174
|
+
method: HTTPMethod,
|
|
175
|
+
target: str,
|
|
176
|
+
headers: dict[str, str],
|
|
177
|
+
backoff: ExponentialBackoff,
|
|
178
|
+
kwargs: dict[str, Any],
|
|
179
|
+
) -> BaseApiResponse:
|
|
138
180
|
while True:
|
|
139
|
-
|
|
140
|
-
raw = await client.request(str(method), target, headers=self.headers, **kwargs)
|
|
141
|
-
|
|
181
|
+
raw = await client.request(str(method), target, headers=headers, **kwargs)
|
|
142
182
|
if raw.status_code not in RETRYABLE_STATUS_CODES or backoff.did_timeout():
|
|
143
183
|
return self._to_response(raw)
|
|
144
184
|
|
|
145
|
-
self.warning(f"{method} {target} returned {raw.status_code}; retrying")
|
|
185
|
+
self.warning(f"{method} {_without_query(target)} returned {raw.status_code}; retrying")
|
|
146
186
|
await backoff.async_wait()
|
|
147
187
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return self.request(HTTPMethod.POST, url, **kwargs)
|
|
153
|
-
|
|
154
|
-
def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
155
|
-
return self.request(HTTPMethod.PUT, url, **kwargs)
|
|
156
|
-
|
|
157
|
-
def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
158
|
-
return self.request(HTTPMethod.PATCH, url, **kwargs)
|
|
159
|
-
|
|
160
|
-
def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
161
|
-
return self.request(HTTPMethod.DELETE, url, **kwargs)
|
|
162
|
-
|
|
163
|
-
async def async_get(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
164
|
-
return await self.async_request(HTTPMethod.GET, url, **kwargs)
|
|
165
|
-
|
|
166
|
-
async def async_post(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
167
|
-
return await self.async_request(HTTPMethod.POST, url, **kwargs)
|
|
168
|
-
|
|
169
|
-
async def async_put(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
170
|
-
return await self.async_request(HTTPMethod.PUT, url, **kwargs)
|
|
188
|
+
@asynccontextmanager
|
|
189
|
+
async def stream_request(self, method: HTTPMethod, url: str, **kwargs: Any) -> AsyncIterator[httpx.Response]:
|
|
190
|
+
"""
|
|
191
|
+
Open a streaming response and yield the raw ``httpx.Response``.
|
|
171
192
|
|
|
172
|
-
|
|
173
|
-
|
|
193
|
+
Holds the connection for the life of the ``async with`` block. Unlike
|
|
194
|
+
``request``, the body is not buffered into a ``BaseApiResponse`` — the
|
|
195
|
+
caller reads it (for example with ``aiter_lines`` for SSE). Retries are
|
|
196
|
+
not applied: a mid-stream failure cannot be replayed safely.
|
|
197
|
+
"""
|
|
198
|
+
if method not in self.supported_methods:
|
|
199
|
+
raise UnsupportedMethodError(method)
|
|
174
200
|
|
|
175
|
-
|
|
176
|
-
|
|
201
|
+
target = self._build_url(url)
|
|
202
|
+
headers = dict(self.headers)
|
|
203
|
+
extra_headers: dict[str, Any] | None = kwargs.pop("headers", None)
|
|
204
|
+
if extra_headers:
|
|
205
|
+
headers.update(extra_headers)
|
|
206
|
+
|
|
207
|
+
if self._http_client is not None:
|
|
208
|
+
async with self._http_client.stream(str(method), target, headers=headers, **kwargs) as response:
|
|
209
|
+
yield response
|
|
210
|
+
return
|
|
211
|
+
|
|
212
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
213
|
+
async with client.stream(str(method), target, headers=headers, **kwargs) as response:
|
|
214
|
+
yield response
|
|
215
|
+
|
|
216
|
+
@allow_sync
|
|
217
|
+
async def get(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
218
|
+
return await self.request(HTTPMethod.GET, url, **kwargs)
|
|
219
|
+
|
|
220
|
+
@allow_sync
|
|
221
|
+
async def post(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
222
|
+
return await self.request(HTTPMethod.POST, url, **kwargs)
|
|
223
|
+
|
|
224
|
+
@allow_sync
|
|
225
|
+
async def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
226
|
+
return await self.request(HTTPMethod.PUT, url, **kwargs)
|
|
227
|
+
|
|
228
|
+
@allow_sync
|
|
229
|
+
async def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
230
|
+
return await self.request(HTTPMethod.PATCH, url, **kwargs)
|
|
231
|
+
|
|
232
|
+
@allow_sync
|
|
233
|
+
async def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
234
|
+
return await self.request(HTTPMethod.DELETE, url, **kwargs)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTTP-specific exceptions.
|
|
3
|
+
|
|
4
|
+
These live here, not in ``corekit.exceptions``, so the exception package does
|
|
5
|
+
not have to import this layer. The bases (``CoreHTTPException`` and the
|
|
6
|
+
retryable / non-retryable variants) still come from there.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from http import HTTPMethod
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from corekit.exceptions.base import (
|
|
13
|
+
InternalCoreException,
|
|
14
|
+
NonRetryableCoreHTTPException,
|
|
15
|
+
RetryableCoreHTTPException,
|
|
16
|
+
)
|
|
17
|
+
from corekit.exceptions.enum import Retryability
|
|
18
|
+
from corekit.http.status import HTTPStatusCode
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"BadGatewayException",
|
|
22
|
+
"BadRequestException",
|
|
23
|
+
"ConflictErrorException",
|
|
24
|
+
"ForbiddenException",
|
|
25
|
+
"GatewayTimeoutException",
|
|
26
|
+
"InternalServerErrorException",
|
|
27
|
+
"NotFoundException",
|
|
28
|
+
"ServiceUnavailableException",
|
|
29
|
+
"TooManyRequestsException",
|
|
30
|
+
"UnauthorizedException",
|
|
31
|
+
"UnprocessableEntityException",
|
|
32
|
+
"UnsupportedMediaTypeException",
|
|
33
|
+
"UnsupportedMethodError",
|
|
34
|
+
"URLMismatchError",
|
|
35
|
+
"RETRYABLE_STATUS_CODES",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _CodedHTTPException:
|
|
40
|
+
"""
|
|
41
|
+
Sets ``status_code`` from the class. The code is the type, so a caller
|
|
42
|
+
cannot pass a different one and have a ``GatewayTimeoutException`` that is
|
|
43
|
+
actually a 200.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
code: HTTPStatusCode
|
|
47
|
+
|
|
48
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
49
|
+
if "status_code" in kwargs:
|
|
50
|
+
raise TypeError(f"{type(self).__name__} does not accept status_code; it is always {int(self.code)}")
|
|
51
|
+
super().__init__(status_code=self.code, **kwargs)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class BadRequestException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
55
|
+
code = HTTPStatusCode.BAD_REQUEST
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class UnauthorizedException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
59
|
+
code = HTTPStatusCode.UNAUTHORIZED
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ForbiddenException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
63
|
+
code = HTTPStatusCode.FORBIDDEN
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class NotFoundException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
67
|
+
code = HTTPStatusCode.NOT_FOUND
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ConflictErrorException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
71
|
+
code = HTTPStatusCode.CONFLICT
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class UnsupportedMediaTypeException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
75
|
+
code = HTTPStatusCode.UNSUPPORTED_MEDIA_TYPE
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class UnprocessableEntityException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
79
|
+
code = HTTPStatusCode.UNPROCESSABLE_ENTITY
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TooManyRequestsException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
83
|
+
code = HTTPStatusCode.TOO_MANY_REQUESTS
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class InternalServerErrorException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
87
|
+
code = HTTPStatusCode.INTERNAL_SERVER_ERROR
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class BadGatewayException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
91
|
+
code = HTTPStatusCode.BAD_GATEWAY
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ServiceUnavailableException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
95
|
+
code = HTTPStatusCode.SERVICE_UNAVAILABLE
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class GatewayTimeoutException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
99
|
+
code = HTTPStatusCode.GATEWAY_TIMEOUT
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# Statuses worth retrying: the server is busy or briefly unavailable, so the
|
|
103
|
+
# same request may well succeed shortly. A 4xx other than 429 will not.
|
|
104
|
+
# Derived from the typed exceptions so this set cannot drift from retryability.
|
|
105
|
+
RETRYABLE_STATUS_CODES = frozenset(
|
|
106
|
+
{
|
|
107
|
+
TooManyRequestsException.code,
|
|
108
|
+
InternalServerErrorException.code,
|
|
109
|
+
BadGatewayException.code,
|
|
110
|
+
ServiceUnavailableException.code,
|
|
111
|
+
GatewayTimeoutException.code,
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class URLMismatchError(InternalCoreException):
|
|
117
|
+
"""
|
|
118
|
+
Raised when an absolute URL does not share the client's base URL.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(self, url: str, base_url: str) -> None:
|
|
122
|
+
super().__init__(
|
|
123
|
+
message=f"URL {url} does not match base URL {base_url}",
|
|
124
|
+
retryable=Retryability.NON_RETRYABLE,
|
|
125
|
+
)
|
|
126
|
+
self.url = url
|
|
127
|
+
self.base_url = base_url
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class UnsupportedMethodError(InternalCoreException):
|
|
131
|
+
"""
|
|
132
|
+
Raised when a client is asked for an HTTP method it does not implement.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
def __init__(self, method: HTTPMethod) -> None:
|
|
136
|
+
super().__init__(
|
|
137
|
+
message=f"Method {method} is not supported",
|
|
138
|
+
retryable=Retryability.NON_RETRYABLE,
|
|
139
|
+
)
|
|
140
|
+
self.method = method
|
corekit/http/response.py
CHANGED
|
@@ -1,12 +1,61 @@
|
|
|
1
1
|
from pydantic import BaseModel, Field
|
|
2
2
|
|
|
3
|
+
from corekit.exceptions import CoreHTTPException
|
|
4
|
+
from corekit.http.exceptions import (
|
|
5
|
+
BadGatewayException,
|
|
6
|
+
ConflictErrorException,
|
|
7
|
+
ForbiddenException,
|
|
8
|
+
GatewayTimeoutException,
|
|
9
|
+
InternalServerErrorException,
|
|
10
|
+
NotFoundException,
|
|
11
|
+
ServiceUnavailableException,
|
|
12
|
+
TooManyRequestsException,
|
|
13
|
+
UnauthorizedException,
|
|
14
|
+
UnprocessableEntityException,
|
|
15
|
+
)
|
|
16
|
+
from corekit.http.status import HTTPStatusCode
|
|
3
17
|
from corekit.schemas.pydantic.fields import DefaultDictField
|
|
4
18
|
from corekit.schemas.types import AnyDict
|
|
5
19
|
|
|
6
20
|
|
|
7
21
|
class BaseApiResponse(BaseModel):
|
|
8
|
-
status_code: int = Field(..., description="HTTP status code")
|
|
22
|
+
status_code: HTTPStatusCode | int = Field(..., description="HTTP status code")
|
|
9
23
|
text: str = Field(..., description="Response text")
|
|
10
24
|
data: AnyDict = DefaultDictField()
|
|
11
25
|
headers: AnyDict = DefaultDictField()
|
|
12
26
|
cookies: AnyDict = DefaultDictField()
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def ok(self) -> bool:
|
|
30
|
+
"""
|
|
31
|
+
Whether the status is below 400. A failure is still a response.
|
|
32
|
+
"""
|
|
33
|
+
return int(self.status_code) < 400
|
|
34
|
+
|
|
35
|
+
def raise_for_status(self) -> "BaseApiResponse":
|
|
36
|
+
"""
|
|
37
|
+
Raise a typed HTTP exception for a failure status.
|
|
38
|
+
|
|
39
|
+
Opt-in. ``request`` still returns the failure so existing callers that
|
|
40
|
+
read ``status_code`` keep working.
|
|
41
|
+
"""
|
|
42
|
+
if self.ok:
|
|
43
|
+
return self
|
|
44
|
+
|
|
45
|
+
typed = {
|
|
46
|
+
int(UnauthorizedException.code): UnauthorizedException,
|
|
47
|
+
int(ForbiddenException.code): ForbiddenException,
|
|
48
|
+
int(NotFoundException.code): NotFoundException,
|
|
49
|
+
int(ConflictErrorException.code): ConflictErrorException,
|
|
50
|
+
int(UnprocessableEntityException.code): UnprocessableEntityException,
|
|
51
|
+
int(TooManyRequestsException.code): TooManyRequestsException,
|
|
52
|
+
int(InternalServerErrorException.code): InternalServerErrorException,
|
|
53
|
+
int(BadGatewayException.code): BadGatewayException,
|
|
54
|
+
int(ServiceUnavailableException.code): ServiceUnavailableException,
|
|
55
|
+
int(GatewayTimeoutException.code): GatewayTimeoutException,
|
|
56
|
+
}
|
|
57
|
+
detail = self.text or f"HTTP {int(self.status_code)}"
|
|
58
|
+
exception_type = typed.get(int(self.status_code))
|
|
59
|
+
if exception_type is None:
|
|
60
|
+
raise CoreHTTPException(status_code=int(self.status_code), detail=detail)
|
|
61
|
+
raise exception_type(detail=detail)
|
corekit/http/status.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTTP codes
|
|
3
|
+
See HTTP Status Code Registry:
|
|
4
|
+
https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
|
|
5
|
+
|
|
6
|
+
And RFC 2324 - https://tools.ietf.org/html/rfc2324
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from corekit.schemas import IntegerEnum
|
|
10
|
+
|
|
11
|
+
__all__ = ["HTTPStatusCode"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class HTTPStatusCode(IntegerEnum):
|
|
15
|
+
CONTINUE = 100
|
|
16
|
+
SWITCHING_PROTOCOLS = 101
|
|
17
|
+
PROCESSING = 102
|
|
18
|
+
EARLY_HINTS = 103
|
|
19
|
+
OK = 200
|
|
20
|
+
CREATED = 201
|
|
21
|
+
ACCEPTED = 202
|
|
22
|
+
NON_AUTHORITATIVE_INFORMATION = 203
|
|
23
|
+
NO_CONTENT = 204
|
|
24
|
+
RESET_CONTENT = 205
|
|
25
|
+
PARTIAL_CONTENT = 206
|
|
26
|
+
MULTI_STATUS = 207
|
|
27
|
+
ALREADY_REPORTED = 208
|
|
28
|
+
IM_USED = 226
|
|
29
|
+
MULTIPLE_CHOICES = 300
|
|
30
|
+
MOVED_PERMANENTLY = 301
|
|
31
|
+
FOUND = 302
|
|
32
|
+
SEE_OTHER = 303
|
|
33
|
+
NOT_MODIFIED = 304
|
|
34
|
+
USE_PROXY = 305
|
|
35
|
+
RESERVED = 306
|
|
36
|
+
TEMPORARY_REDIRECT = 307
|
|
37
|
+
PERMANENT_REDIRECT = 308
|
|
38
|
+
BAD_REQUEST = 400
|
|
39
|
+
UNAUTHORIZED = 401
|
|
40
|
+
PAYMENT_REQUIRED = 402
|
|
41
|
+
FORBIDDEN = 403
|
|
42
|
+
NOT_FOUND = 404
|
|
43
|
+
METHOD_NOT_ALLOWED = 405
|
|
44
|
+
NOT_ACCEPTABLE = 406
|
|
45
|
+
PROXY_AUTHENTICATION_REQUIRED = 407
|
|
46
|
+
REQUEST_TIMEOUT = 408
|
|
47
|
+
CONFLICT = 409
|
|
48
|
+
GONE = 410
|
|
49
|
+
LENGTH_REQUIRED = 411
|
|
50
|
+
PRECONDITION_FAILED = 412
|
|
51
|
+
REQUEST_ENTITY_TOO_LARGE = 413
|
|
52
|
+
REQUEST_URI_TOO_LONG = 414
|
|
53
|
+
UNSUPPORTED_MEDIA_TYPE = 415
|
|
54
|
+
REQUESTED_RANGE_NOT_SATISFIABLE = 416
|
|
55
|
+
EXPECTATION_FAILED = 417
|
|
56
|
+
IM_A_TEAPOT = 418
|
|
57
|
+
MISDIRECTED_REQUEST = 421
|
|
58
|
+
UNPROCESSABLE_ENTITY = 422
|
|
59
|
+
LOCKED = 423
|
|
60
|
+
FAILED_DEPENDENCY = 424
|
|
61
|
+
TOO_EARLY = 425
|
|
62
|
+
UPGRADE_REQUIRED = 426
|
|
63
|
+
PRECONDITION_REQUIRED = 428
|
|
64
|
+
TOO_MANY_REQUESTS = 429
|
|
65
|
+
REQUEST_HEADER_FIELDS_TOO_LARGE = 431
|
|
66
|
+
UNAVAILABLE_FOR_LEGAL_REASONS = 451
|
|
67
|
+
INTERNAL_SERVER_ERROR = 500
|
|
68
|
+
NOT_IMPLEMENTED = 501
|
|
69
|
+
BAD_GATEWAY = 502
|
|
70
|
+
SERVICE_UNAVAILABLE = 503
|
|
71
|
+
GATEWAY_TIMEOUT = 504
|
|
72
|
+
HTTP_VERSION_NOT_SUPPORTED = 505
|
|
73
|
+
VARIANT_ALSO_NEGOTIATES = 506
|
|
74
|
+
INSUFFICIENT_STORAGE = 507
|
|
75
|
+
LOOP_DETECTED = 508
|
|
76
|
+
NOT_EXTENDED = 510
|
|
77
|
+
NETWORK_AUTHENTICATION_REQUIRED = 511
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def coerce(cls, code: int) -> "HTTPStatusCode | int":
|
|
81
|
+
"""
|
|
82
|
+
Return the enum member for a known status, or the raw code otherwise.
|
|
83
|
+
|
|
84
|
+
Proxies and CDNs emit statuses this registry does not name. Callers
|
|
85
|
+
should still get the numeric code rather than a failed response parse.
|
|
86
|
+
"""
|
|
87
|
+
if cls.is_member(code):
|
|
88
|
+
return cls.from_int(code)
|
|
89
|
+
return code
|
corekit/http/stream.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Decode streaming HTTP response bodies into typed events.
|
|
3
|
+
|
|
4
|
+
``BaseHttpClient.stream_request`` yields a raw ``httpx.Response``. Decoders
|
|
5
|
+
turn that into an async iterator of useful values (SSE text, SSE JSON, …).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from collections.abc import AsyncIterator
|
|
13
|
+
from typing import Any, Generic, TypeVar
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from corekit.observability import Loggable
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"SseDecoder",
|
|
21
|
+
"SseJsonDecoder",
|
|
22
|
+
"SseTextDecoder",
|
|
23
|
+
"StreamDecoder",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
T = TypeVar("T")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class StreamDecoder(Loggable, ABC, Generic[T]):
|
|
30
|
+
"""
|
|
31
|
+
Turn a streaming ``httpx.Response`` into an async iterator of ``T``.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
def decode(self, response: httpx.Response) -> AsyncIterator[T]:
|
|
36
|
+
"""
|
|
37
|
+
Consume ``response`` and yield decoded items.
|
|
38
|
+
|
|
39
|
+
Implemented as an async generator on concrete subclasses.
|
|
40
|
+
"""
|
|
41
|
+
...
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class SseDecoder(StreamDecoder[T], ABC):
|
|
45
|
+
"""
|
|
46
|
+
Server-Sent Events framing: ``data:`` lines, skip comments / blanks.
|
|
47
|
+
|
|
48
|
+
Optional ``done_sentinel`` (e.g. OpenAI's ``[DONE]``) ends the stream
|
|
49
|
+
early. Subclasses interpret each data payload via ``_decode_payload``.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(self, *, done_sentinel: str | None = None) -> None:
|
|
53
|
+
super().__init__()
|
|
54
|
+
self.done_sentinel = done_sentinel
|
|
55
|
+
|
|
56
|
+
async def decode(self, response: httpx.Response) -> AsyncIterator[T]:
|
|
57
|
+
async for data in self._iter_data_payloads(response):
|
|
58
|
+
item = self._decode_payload(data)
|
|
59
|
+
if item is not None:
|
|
60
|
+
yield item
|
|
61
|
+
|
|
62
|
+
async def _iter_data_payloads(self, response: httpx.Response) -> AsyncIterator[str]:
|
|
63
|
+
async for line in response.aiter_lines():
|
|
64
|
+
line = line.strip()
|
|
65
|
+
if not line.startswith("data:"):
|
|
66
|
+
continue
|
|
67
|
+
data = line.removeprefix("data:").strip()
|
|
68
|
+
if not data:
|
|
69
|
+
continue
|
|
70
|
+
if self.done_sentinel is not None and data == self.done_sentinel:
|
|
71
|
+
return
|
|
72
|
+
yield data
|
|
73
|
+
|
|
74
|
+
@abstractmethod
|
|
75
|
+
def _decode_payload(self, data: str) -> T | None:
|
|
76
|
+
"""
|
|
77
|
+
Map one SSE data payload to ``T``, or ``None`` to skip.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class SseTextDecoder(SseDecoder[str]):
|
|
82
|
+
"""
|
|
83
|
+
Yield SSE data payloads as plain strings.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def _decode_payload(self, data: str) -> str | None:
|
|
87
|
+
return data
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class SseJsonDecoder(SseDecoder[dict[str, Any]]):
|
|
91
|
+
"""
|
|
92
|
+
Yield SSE data payloads parsed as JSON objects.
|
|
93
|
+
|
|
94
|
+
Non-object JSON and decode failures are skipped (with a warning). Default
|
|
95
|
+
``done_sentinel`` matches OpenAI-compatible chat streams.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(self, *, done_sentinel: str | None = "[DONE]") -> None:
|
|
99
|
+
super().__init__(done_sentinel=done_sentinel)
|
|
100
|
+
|
|
101
|
+
def _decode_payload(self, data: str) -> dict[str, Any] | None:
|
|
102
|
+
try:
|
|
103
|
+
payload = json.loads(data)
|
|
104
|
+
except json.JSONDecodeError:
|
|
105
|
+
self.warning(f"[SseJsonDecoder] Skipping bad SSE payload: {data!r}")
|
|
106
|
+
return None
|
|
107
|
+
if isinstance(payload, dict):
|
|
108
|
+
return payload
|
|
109
|
+
self.warning(f"[SseJsonDecoder] Skipping non-object SSE JSON: {type(payload).__name__}")
|
|
110
|
+
return None
|