python-corekit 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/api/application.py +47 -9
- corekit/api/lifespan.py +26 -3
- corekit/concurrency/__init__.py +2 -2
- corekit/concurrency/decorators.py +32 -5
- corekit/concurrency/thread_local.py +2 -2
- corekit/concurrency/worker.py +9 -0
- corekit/config/loader.py +42 -5
- corekit/config/settings.py +11 -1
- corekit/connections/__init__.py +7 -1
- corekit/connections/connectable.py +45 -4
- corekit/connections/redis/connection.py +53 -10
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +39 -5
- corekit/connections/sql/fields/__init__.py +2 -2
- corekit/connections/sql/fields/jsonb.py +13 -6
- corekit/connections/sql/migration/__init__.py +4 -0
- corekit/connections/sql/migration/operations.py +69 -2
- corekit/connections/sql/operations/base.py +11 -2
- corekit/connections/sql/operations/statements.py +25 -5
- corekit/connections/sql/table.py +7 -29
- corekit/crypto/__init__.py +3 -1
- corekit/crypto/constants.py +2 -2
- corekit/crypto/hasher.py +9 -4
- corekit/data/dataset.py +8 -2
- corekit/data/expressions/__init__.py +3 -3
- corekit/data/expressions/comparison.py +19 -80
- corekit/data/expressions/expression.py +0 -32
- corekit/data/expressions/operator.py +13 -28
- corekit/data/stats.py +3 -0
- corekit/decorators/exception_handling.py +36 -8
- corekit/docker/watchdog.py +50 -31
- corekit/etl/__init__.py +2 -1
- corekit/etl/connection.py +14 -12
- corekit/etl/extract/extractor.py +6 -13
- corekit/etl/orchestrator.py +19 -2
- corekit/etl/schemas.py +2 -2
- corekit/etl/transform/transformer.py +4 -1
- corekit/events/publisher.py +1 -1
- corekit/events/reader.py +26 -21
- corekit/events/sse.py +4 -1
- corekit/events/websocket.py +24 -11
- corekit/exceptions/__init__.py +24 -9
- corekit/exceptions/base.py +139 -10
- corekit/exceptions/enum.py +17 -0
- corekit/exceptions/types.py +6 -6
- corekit/files/__init__.py +2 -4
- corekit/files/base.py +15 -2
- corekit/files/enum.py +0 -5
- corekit/files/json.py +16 -2
- corekit/http/__init__.py +43 -5
- corekit/http/api.py +24 -0
- corekit/http/client.py +100 -73
- corekit/http/exceptions.py +140 -0
- corekit/http/response.py +50 -1
- corekit/http/status.py +89 -0
- corekit/jobs/runner.py +12 -1
- corekit/jobs/task.py +23 -2
- corekit/log_monitor/models.py +8 -2
- corekit/log_monitor/service.py +77 -38
- corekit/notifications/base.py +18 -10
- corekit/observability/__init__.py +9 -2
- corekit/observability/benchmarkable.py +23 -5
- corekit/observability/loggable.py +21 -0
- corekit/observability/request_context.py +55 -2
- corekit/observability/timing/timer.py +4 -2
- corekit/registry/__init__.py +2 -2
- corekit/registry/registry.py +55 -14
- corekit/schemas/enum.py +22 -1
- corekit/schemas/types.py +6 -1
- corekit/serialization/__init__.py +2 -0
- corekit/serialization/pickle_file.py +61 -0
- corekit/serialization/serializable.py +22 -2
- corekit/serialization/serializer.py +9 -2
- corekit/utils/collections.py +22 -13
- corekit/utils/payload.py +12 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/METADATA +7 -7
- python_corekit-0.3.0.dist-info/RECORD +145 -0
- corekit/constants.py +0 -45
- corekit/exceptions/http/exceptions.py +0 -37
- corekit/files/pickle.py +0 -12
- python_corekit-0.2.0.dist-info/RECORD +0 -143
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/WHEEL +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/top_level.txt +0 -0
corekit/http/client.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
"""
|
|
2
2
|
A small HTTP client with retries.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
``BaseHttpClient`` is the transport: relative paths, optional strict URL
|
|
5
|
+
checking, and retries for statuses the typed HTTP exceptions mark as
|
|
6
|
+
retryable. Subclass ``BaseApiClient`` when the client always talks to one
|
|
7
|
+
API and a foreign absolute URL should be a bug.
|
|
5
8
|
|
|
6
9
|
class GithubClient(BaseApiClient):
|
|
7
10
|
'''
|
|
@@ -17,46 +20,79 @@ Subclass and give it a base URL::
|
|
|
17
20
|
return {"Authorization": f"Bearer {self.token}"}
|
|
18
21
|
|
|
19
22
|
client = GithubClient()
|
|
20
|
-
response = client.get("/user") # or await client.
|
|
23
|
+
response = client.get("/user") # or await client.get("/user")
|
|
21
24
|
response.data["login"]
|
|
22
25
|
|
|
23
26
|
Every response comes back as a ``BaseApiResponse``, so callers see one shape
|
|
24
|
-
regardless of what the endpoint returned.
|
|
25
|
-
status are retried with exponential backoff.
|
|
27
|
+
regardless of what the endpoint returned.
|
|
26
28
|
"""
|
|
27
29
|
|
|
28
30
|
from http import HTTPMethod
|
|
29
31
|
from typing import Any
|
|
30
|
-
from urllib.parse import urljoin
|
|
32
|
+
from urllib.parse import urljoin, urlsplit, urlunsplit
|
|
31
33
|
|
|
32
34
|
import httpx
|
|
33
35
|
|
|
36
|
+
from corekit.concurrency.decorators import allow_sync
|
|
37
|
+
from corekit.http.exceptions import RETRYABLE_STATUS_CODES, UnsupportedMethodError, URLMismatchError
|
|
34
38
|
from corekit.http.exponential_backoff import ExponentialBackoff
|
|
35
39
|
from corekit.http.response import BaseApiResponse
|
|
40
|
+
from corekit.http.status import HTTPStatusCode
|
|
36
41
|
from corekit.observability.benchmarkable import Benchmarkable
|
|
37
42
|
|
|
38
|
-
__all__ = ["
|
|
43
|
+
__all__ = ["BaseHttpClient"]
|
|
39
44
|
|
|
40
|
-
# Statuses worth retrying: the server is busy or briefly unavailable, so the
|
|
41
|
-
# same request may well succeed shortly. A 4xx other than 429 will not.
|
|
42
|
-
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})
|
|
43
45
|
|
|
44
|
-
|
|
45
|
-
class URLMismatchError(Exception):
|
|
46
|
+
def _without_query(url: str) -> str:
|
|
46
47
|
"""
|
|
47
|
-
|
|
48
|
+
Drop the query string before a retry warning is logged.
|
|
48
49
|
"""
|
|
50
|
+
parts = urlsplit(url)
|
|
51
|
+
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
|
52
|
+
|
|
49
53
|
|
|
54
|
+
_DEFAULT_METHODS = frozenset(
|
|
55
|
+
{
|
|
56
|
+
HTTPMethod.GET,
|
|
57
|
+
HTTPMethod.POST,
|
|
58
|
+
HTTPMethod.PUT,
|
|
59
|
+
HTTPMethod.PATCH,
|
|
60
|
+
HTTPMethod.DELETE,
|
|
61
|
+
}
|
|
62
|
+
)
|
|
50
63
|
|
|
51
|
-
|
|
64
|
+
|
|
65
|
+
class BaseHttpClient(Benchmarkable):
|
|
52
66
|
"""
|
|
53
|
-
Base for
|
|
67
|
+
Base for HTTP clients. Override ``base_url`` and, usually, ``headers``.
|
|
68
|
+
|
|
69
|
+
``strict=False`` accepts a foreign absolute URL as-is. ``BaseApiClient``
|
|
70
|
+
turns that into an error.
|
|
54
71
|
"""
|
|
55
72
|
|
|
56
|
-
def __init__(
|
|
73
|
+
def __init__(
|
|
74
|
+
self,
|
|
75
|
+
retries: int = 3,
|
|
76
|
+
timeout: float = 30.0,
|
|
77
|
+
strict: bool = False,
|
|
78
|
+
base_delay: float = 1,
|
|
79
|
+
http_client: httpx.AsyncClient | None = None,
|
|
80
|
+
) -> None:
|
|
57
81
|
super().__init__()
|
|
58
82
|
self.retries = retries
|
|
59
83
|
self.timeout = timeout
|
|
84
|
+
self.strict = strict
|
|
85
|
+
self.base_delay = base_delay
|
|
86
|
+
# Optional pool. Left unset by default: @allow_sync runs sync calls on a
|
|
87
|
+
# throwaway loop, and an AsyncClient cannot move between loops.
|
|
88
|
+
self._http_client = http_client
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def supported_methods(self) -> frozenset[HTTPMethod]:
|
|
92
|
+
"""
|
|
93
|
+
HTTP methods this client will send. Override to narrow or extend the set.
|
|
94
|
+
"""
|
|
95
|
+
return _DEFAULT_METHODS
|
|
60
96
|
|
|
61
97
|
@property
|
|
62
98
|
def headers(self) -> dict[str, str]:
|
|
@@ -77,16 +113,18 @@ class BaseApiClient(Benchmarkable):
|
|
|
77
113
|
Resolve a path against the base URL, accepting an absolute URL that
|
|
78
114
|
already matches it.
|
|
79
115
|
"""
|
|
80
|
-
if url.startswith(self.base_url):
|
|
116
|
+
if self.base_url and url.startswith(self.base_url):
|
|
81
117
|
return url
|
|
82
118
|
if url.startswith("http"):
|
|
83
|
-
|
|
119
|
+
if self.strict:
|
|
120
|
+
raise URLMismatchError(url, self.base_url)
|
|
121
|
+
return url
|
|
84
122
|
return urljoin(self.base_url, url)
|
|
85
123
|
|
|
86
124
|
@staticmethod
|
|
87
|
-
def _to_response(raw:
|
|
125
|
+
def _to_response(raw: httpx.Response) -> BaseApiResponse:
|
|
88
126
|
"""
|
|
89
|
-
Normalize
|
|
127
|
+
Normalize a httpx response into a BaseApiResponse.
|
|
90
128
|
|
|
91
129
|
The body is decoded as JSON when it parses, and left as text otherwise,
|
|
92
130
|
so a caller never has to guard against a non-JSON error page.
|
|
@@ -99,78 +137,67 @@ class BaseApiClient(Benchmarkable):
|
|
|
99
137
|
data = {"data": data}
|
|
100
138
|
|
|
101
139
|
return BaseApiResponse(
|
|
102
|
-
status_code=raw.status_code,
|
|
140
|
+
status_code=HTTPStatusCode.coerce(raw.status_code),
|
|
103
141
|
text=raw.text,
|
|
104
142
|
data=data,
|
|
105
143
|
headers=dict(raw.headers),
|
|
106
144
|
cookies=dict(raw.cookies),
|
|
107
145
|
)
|
|
108
146
|
|
|
109
|
-
def
|
|
110
|
-
return ExponentialBackoff(retries=self.retries)
|
|
111
|
-
|
|
112
|
-
def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
113
|
-
"""
|
|
114
|
-
Send a request, retrying retryable statuses.
|
|
115
|
-
"""
|
|
116
|
-
|
|
117
|
-
target = self._build_url(url)
|
|
118
|
-
backoff = self._new_backoff()
|
|
119
|
-
|
|
120
|
-
while True:
|
|
121
|
-
with httpx.Client(timeout=self.timeout) as client:
|
|
122
|
-
raw = client.request(str(method), target, headers=self.headers, **kwargs)
|
|
123
|
-
|
|
124
|
-
if raw.status_code not in RETRYABLE_STATUS_CODES or backoff.did_timeout():
|
|
125
|
-
return self._to_response(raw)
|
|
126
|
-
|
|
127
|
-
self.warning(f"{method} {target} returned {raw.status_code}; retrying")
|
|
128
|
-
backoff.wait()
|
|
129
|
-
|
|
130
|
-
async def async_request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
147
|
+
async def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
131
148
|
"""
|
|
132
149
|
Send a request asynchronously, retrying retryable statuses.
|
|
133
150
|
"""
|
|
151
|
+
if method not in self.supported_methods:
|
|
152
|
+
raise UnsupportedMethodError(method)
|
|
134
153
|
|
|
135
154
|
target = self._build_url(url)
|
|
136
|
-
|
|
137
|
-
|
|
155
|
+
headers = dict(self.headers)
|
|
156
|
+
extra_headers = kwargs.pop("headers", None)
|
|
157
|
+
if extra_headers:
|
|
158
|
+
headers.update(extra_headers)
|
|
159
|
+
# Fresh each request: ExponentialBackoff keeps its own attempt count.
|
|
160
|
+
backoff = ExponentialBackoff(retries=self.retries, base_delay=self.base_delay)
|
|
161
|
+
|
|
162
|
+
if self._http_client is not None:
|
|
163
|
+
return await self._exchange(self._http_client, method, target, headers, backoff, kwargs)
|
|
164
|
+
|
|
165
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
166
|
+
return await self._exchange(client, method, target, headers, backoff, kwargs)
|
|
167
|
+
|
|
168
|
+
async def _exchange(
|
|
169
|
+
self,
|
|
170
|
+
client: httpx.AsyncClient,
|
|
171
|
+
method: HTTPMethod,
|
|
172
|
+
target: str,
|
|
173
|
+
headers: dict[str, str],
|
|
174
|
+
backoff: ExponentialBackoff,
|
|
175
|
+
kwargs: dict[str, Any],
|
|
176
|
+
) -> BaseApiResponse:
|
|
138
177
|
while True:
|
|
139
|
-
|
|
140
|
-
raw = await client.request(str(method), target, headers=self.headers, **kwargs)
|
|
141
|
-
|
|
178
|
+
raw = await client.request(str(method), target, headers=headers, **kwargs)
|
|
142
179
|
if raw.status_code not in RETRYABLE_STATUS_CODES or backoff.did_timeout():
|
|
143
180
|
return self._to_response(raw)
|
|
144
181
|
|
|
145
|
-
self.warning(f"{method} {target} returned {raw.status_code}; retrying")
|
|
182
|
+
self.warning(f"{method} {_without_query(target)} returned {raw.status_code}; retrying")
|
|
146
183
|
await backoff.async_wait()
|
|
147
184
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def post(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
152
|
-
return self.request(HTTPMethod.POST, url, **kwargs)
|
|
153
|
-
|
|
154
|
-
def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
155
|
-
return self.request(HTTPMethod.PUT, url, **kwargs)
|
|
156
|
-
|
|
157
|
-
def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
158
|
-
return self.request(HTTPMethod.PATCH, url, **kwargs)
|
|
159
|
-
|
|
160
|
-
def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
161
|
-
return self.request(HTTPMethod.DELETE, url, **kwargs)
|
|
162
|
-
|
|
163
|
-
async def async_get(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
164
|
-
return await self.async_request(HTTPMethod.GET, url, **kwargs)
|
|
185
|
+
@allow_sync
|
|
186
|
+
async def get(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
187
|
+
return await self.request(HTTPMethod.GET, url, **kwargs)
|
|
165
188
|
|
|
166
|
-
|
|
167
|
-
|
|
189
|
+
@allow_sync
|
|
190
|
+
async def post(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
191
|
+
return await self.request(HTTPMethod.POST, url, **kwargs)
|
|
168
192
|
|
|
169
|
-
|
|
170
|
-
|
|
193
|
+
@allow_sync
|
|
194
|
+
async def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
195
|
+
return await self.request(HTTPMethod.PUT, url, **kwargs)
|
|
171
196
|
|
|
172
|
-
|
|
173
|
-
|
|
197
|
+
@allow_sync
|
|
198
|
+
async def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
199
|
+
return await self.request(HTTPMethod.PATCH, url, **kwargs)
|
|
174
200
|
|
|
175
|
-
|
|
176
|
-
|
|
201
|
+
@allow_sync
|
|
202
|
+
async def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
|
|
203
|
+
return await self.request(HTTPMethod.DELETE, url, **kwargs)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTTP-specific exceptions.
|
|
3
|
+
|
|
4
|
+
These live here, not in ``corekit.exceptions``, so the exception package does
|
|
5
|
+
not have to import this layer. The bases (``CoreHTTPException`` and the
|
|
6
|
+
retryable / non-retryable variants) still come from there.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from http import HTTPMethod
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from corekit.exceptions.base import (
|
|
13
|
+
InternalCoreException,
|
|
14
|
+
NonRetryableCoreHTTPException,
|
|
15
|
+
RetryableCoreHTTPException,
|
|
16
|
+
)
|
|
17
|
+
from corekit.exceptions.enum import Retryability
|
|
18
|
+
from corekit.http.status import HTTPStatusCode
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"BadGatewayException",
|
|
22
|
+
"BadRequestException",
|
|
23
|
+
"ConflictErrorException",
|
|
24
|
+
"ForbiddenException",
|
|
25
|
+
"GatewayTimeoutException",
|
|
26
|
+
"InternalServerErrorException",
|
|
27
|
+
"NotFoundException",
|
|
28
|
+
"ServiceUnavailableException",
|
|
29
|
+
"TooManyRequestsException",
|
|
30
|
+
"UnauthorizedException",
|
|
31
|
+
"UnprocessableEntityException",
|
|
32
|
+
"UnsupportedMediaTypeException",
|
|
33
|
+
"UnsupportedMethodError",
|
|
34
|
+
"URLMismatchError",
|
|
35
|
+
"RETRYABLE_STATUS_CODES",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _CodedHTTPException:
|
|
40
|
+
"""
|
|
41
|
+
Sets ``status_code`` from the class. The code is the type, so a caller
|
|
42
|
+
cannot pass a different one and have a ``GatewayTimeoutException`` that is
|
|
43
|
+
actually a 200.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
code: HTTPStatusCode
|
|
47
|
+
|
|
48
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
49
|
+
if "status_code" in kwargs:
|
|
50
|
+
raise TypeError(f"{type(self).__name__} does not accept status_code; it is always {int(self.code)}")
|
|
51
|
+
super().__init__(status_code=self.code, **kwargs)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class BadRequestException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
55
|
+
code = HTTPStatusCode.BAD_REQUEST
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class UnauthorizedException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
59
|
+
code = HTTPStatusCode.UNAUTHORIZED
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ForbiddenException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
63
|
+
code = HTTPStatusCode.FORBIDDEN
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class NotFoundException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
67
|
+
code = HTTPStatusCode.NOT_FOUND
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ConflictErrorException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
71
|
+
code = HTTPStatusCode.CONFLICT
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class UnsupportedMediaTypeException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
75
|
+
code = HTTPStatusCode.UNSUPPORTED_MEDIA_TYPE
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class UnprocessableEntityException(_CodedHTTPException, NonRetryableCoreHTTPException):
|
|
79
|
+
code = HTTPStatusCode.UNPROCESSABLE_ENTITY
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TooManyRequestsException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
83
|
+
code = HTTPStatusCode.TOO_MANY_REQUESTS
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class InternalServerErrorException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
87
|
+
code = HTTPStatusCode.INTERNAL_SERVER_ERROR
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class BadGatewayException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
91
|
+
code = HTTPStatusCode.BAD_GATEWAY
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ServiceUnavailableException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
95
|
+
code = HTTPStatusCode.SERVICE_UNAVAILABLE
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class GatewayTimeoutException(_CodedHTTPException, RetryableCoreHTTPException):
|
|
99
|
+
code = HTTPStatusCode.GATEWAY_TIMEOUT
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# Statuses worth retrying: the server is busy or briefly unavailable, so the
|
|
103
|
+
# same request may well succeed shortly. A 4xx other than 429 will not.
|
|
104
|
+
# Derived from the typed exceptions so this set cannot drift from retryability.
|
|
105
|
+
RETRYABLE_STATUS_CODES = frozenset(
|
|
106
|
+
{
|
|
107
|
+
TooManyRequestsException.code,
|
|
108
|
+
InternalServerErrorException.code,
|
|
109
|
+
BadGatewayException.code,
|
|
110
|
+
ServiceUnavailableException.code,
|
|
111
|
+
GatewayTimeoutException.code,
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class URLMismatchError(InternalCoreException):
|
|
117
|
+
"""
|
|
118
|
+
Raised when an absolute URL does not share the client's base URL.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(self, url: str, base_url: str) -> None:
|
|
122
|
+
super().__init__(
|
|
123
|
+
message=f"URL {url} does not match base URL {base_url}",
|
|
124
|
+
retryable=Retryability.NON_RETRYABLE,
|
|
125
|
+
)
|
|
126
|
+
self.url = url
|
|
127
|
+
self.base_url = base_url
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class UnsupportedMethodError(InternalCoreException):
|
|
131
|
+
"""
|
|
132
|
+
Raised when a client is asked for an HTTP method it does not implement.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
def __init__(self, method: HTTPMethod) -> None:
|
|
136
|
+
super().__init__(
|
|
137
|
+
message=f"Method {method} is not supported",
|
|
138
|
+
retryable=Retryability.NON_RETRYABLE,
|
|
139
|
+
)
|
|
140
|
+
self.method = method
|
corekit/http/response.py
CHANGED
|
@@ -1,12 +1,61 @@
|
|
|
1
1
|
from pydantic import BaseModel, Field
|
|
2
2
|
|
|
3
|
+
from corekit.exceptions import CoreHTTPException
|
|
4
|
+
from corekit.http.exceptions import (
|
|
5
|
+
BadGatewayException,
|
|
6
|
+
ConflictErrorException,
|
|
7
|
+
ForbiddenException,
|
|
8
|
+
GatewayTimeoutException,
|
|
9
|
+
InternalServerErrorException,
|
|
10
|
+
NotFoundException,
|
|
11
|
+
ServiceUnavailableException,
|
|
12
|
+
TooManyRequestsException,
|
|
13
|
+
UnauthorizedException,
|
|
14
|
+
UnprocessableEntityException,
|
|
15
|
+
)
|
|
16
|
+
from corekit.http.status import HTTPStatusCode
|
|
3
17
|
from corekit.schemas.pydantic.fields import DefaultDictField
|
|
4
18
|
from corekit.schemas.types import AnyDict
|
|
5
19
|
|
|
6
20
|
|
|
7
21
|
class BaseApiResponse(BaseModel):
|
|
8
|
-
status_code: int = Field(..., description="HTTP status code")
|
|
22
|
+
status_code: HTTPStatusCode | int = Field(..., description="HTTP status code")
|
|
9
23
|
text: str = Field(..., description="Response text")
|
|
10
24
|
data: AnyDict = DefaultDictField()
|
|
11
25
|
headers: AnyDict = DefaultDictField()
|
|
12
26
|
cookies: AnyDict = DefaultDictField()
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def ok(self) -> bool:
|
|
30
|
+
"""
|
|
31
|
+
Whether the status is below 400. A failure is still a response.
|
|
32
|
+
"""
|
|
33
|
+
return int(self.status_code) < 400
|
|
34
|
+
|
|
35
|
+
def raise_for_status(self) -> "BaseApiResponse":
|
|
36
|
+
"""
|
|
37
|
+
Raise a typed HTTP exception for a failure status.
|
|
38
|
+
|
|
39
|
+
Opt-in. ``request`` still returns the failure so existing callers that
|
|
40
|
+
read ``status_code`` keep working.
|
|
41
|
+
"""
|
|
42
|
+
if self.ok:
|
|
43
|
+
return self
|
|
44
|
+
|
|
45
|
+
typed = {
|
|
46
|
+
int(UnauthorizedException.code): UnauthorizedException,
|
|
47
|
+
int(ForbiddenException.code): ForbiddenException,
|
|
48
|
+
int(NotFoundException.code): NotFoundException,
|
|
49
|
+
int(ConflictErrorException.code): ConflictErrorException,
|
|
50
|
+
int(UnprocessableEntityException.code): UnprocessableEntityException,
|
|
51
|
+
int(TooManyRequestsException.code): TooManyRequestsException,
|
|
52
|
+
int(InternalServerErrorException.code): InternalServerErrorException,
|
|
53
|
+
int(BadGatewayException.code): BadGatewayException,
|
|
54
|
+
int(ServiceUnavailableException.code): ServiceUnavailableException,
|
|
55
|
+
int(GatewayTimeoutException.code): GatewayTimeoutException,
|
|
56
|
+
}
|
|
57
|
+
detail = self.text or f"HTTP {int(self.status_code)}"
|
|
58
|
+
exception_type = typed.get(int(self.status_code))
|
|
59
|
+
if exception_type is None:
|
|
60
|
+
raise CoreHTTPException(status_code=int(self.status_code), detail=detail)
|
|
61
|
+
raise exception_type(detail=detail)
|
corekit/http/status.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTTP codes
|
|
3
|
+
See HTTP Status Code Registry:
|
|
4
|
+
https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
|
|
5
|
+
|
|
6
|
+
And RFC 2324 - https://tools.ietf.org/html/rfc2324
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from corekit.schemas import IntegerEnum
|
|
10
|
+
|
|
11
|
+
__all__ = ["HTTPStatusCode"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class HTTPStatusCode(IntegerEnum):
|
|
15
|
+
CONTINUE = 100
|
|
16
|
+
SWITCHING_PROTOCOLS = 101
|
|
17
|
+
PROCESSING = 102
|
|
18
|
+
EARLY_HINTS = 103
|
|
19
|
+
OK = 200
|
|
20
|
+
CREATED = 201
|
|
21
|
+
ACCEPTED = 202
|
|
22
|
+
NON_AUTHORITATIVE_INFORMATION = 203
|
|
23
|
+
NO_CONTENT = 204
|
|
24
|
+
RESET_CONTENT = 205
|
|
25
|
+
PARTIAL_CONTENT = 206
|
|
26
|
+
MULTI_STATUS = 207
|
|
27
|
+
ALREADY_REPORTED = 208
|
|
28
|
+
IM_USED = 226
|
|
29
|
+
MULTIPLE_CHOICES = 300
|
|
30
|
+
MOVED_PERMANENTLY = 301
|
|
31
|
+
FOUND = 302
|
|
32
|
+
SEE_OTHER = 303
|
|
33
|
+
NOT_MODIFIED = 304
|
|
34
|
+
USE_PROXY = 305
|
|
35
|
+
RESERVED = 306
|
|
36
|
+
TEMPORARY_REDIRECT = 307
|
|
37
|
+
PERMANENT_REDIRECT = 308
|
|
38
|
+
BAD_REQUEST = 400
|
|
39
|
+
UNAUTHORIZED = 401
|
|
40
|
+
PAYMENT_REQUIRED = 402
|
|
41
|
+
FORBIDDEN = 403
|
|
42
|
+
NOT_FOUND = 404
|
|
43
|
+
METHOD_NOT_ALLOWED = 405
|
|
44
|
+
NOT_ACCEPTABLE = 406
|
|
45
|
+
PROXY_AUTHENTICATION_REQUIRED = 407
|
|
46
|
+
REQUEST_TIMEOUT = 408
|
|
47
|
+
CONFLICT = 409
|
|
48
|
+
GONE = 410
|
|
49
|
+
LENGTH_REQUIRED = 411
|
|
50
|
+
PRECONDITION_FAILED = 412
|
|
51
|
+
REQUEST_ENTITY_TOO_LARGE = 413
|
|
52
|
+
REQUEST_URI_TOO_LONG = 414
|
|
53
|
+
UNSUPPORTED_MEDIA_TYPE = 415
|
|
54
|
+
REQUESTED_RANGE_NOT_SATISFIABLE = 416
|
|
55
|
+
EXPECTATION_FAILED = 417
|
|
56
|
+
IM_A_TEAPOT = 418
|
|
57
|
+
MISDIRECTED_REQUEST = 421
|
|
58
|
+
UNPROCESSABLE_ENTITY = 422
|
|
59
|
+
LOCKED = 423
|
|
60
|
+
FAILED_DEPENDENCY = 424
|
|
61
|
+
TOO_EARLY = 425
|
|
62
|
+
UPGRADE_REQUIRED = 426
|
|
63
|
+
PRECONDITION_REQUIRED = 428
|
|
64
|
+
TOO_MANY_REQUESTS = 429
|
|
65
|
+
REQUEST_HEADER_FIELDS_TOO_LARGE = 431
|
|
66
|
+
UNAVAILABLE_FOR_LEGAL_REASONS = 451
|
|
67
|
+
INTERNAL_SERVER_ERROR = 500
|
|
68
|
+
NOT_IMPLEMENTED = 501
|
|
69
|
+
BAD_GATEWAY = 502
|
|
70
|
+
SERVICE_UNAVAILABLE = 503
|
|
71
|
+
GATEWAY_TIMEOUT = 504
|
|
72
|
+
HTTP_VERSION_NOT_SUPPORTED = 505
|
|
73
|
+
VARIANT_ALSO_NEGOTIATES = 506
|
|
74
|
+
INSUFFICIENT_STORAGE = 507
|
|
75
|
+
LOOP_DETECTED = 508
|
|
76
|
+
NOT_EXTENDED = 510
|
|
77
|
+
NETWORK_AUTHENTICATION_REQUIRED = 511
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def coerce(cls, code: int) -> "HTTPStatusCode | int":
|
|
81
|
+
"""
|
|
82
|
+
Return the enum member for a known status, or the raw code otherwise.
|
|
83
|
+
|
|
84
|
+
Proxies and CDNs emit statuses this registry does not name. Callers
|
|
85
|
+
should still get the numeric code rather than a failed response parse.
|
|
86
|
+
"""
|
|
87
|
+
if cls.is_member(code):
|
|
88
|
+
return cls.from_int(code)
|
|
89
|
+
return code
|
corekit/jobs/runner.py
CHANGED
|
@@ -13,6 +13,8 @@ Enqueueing a bound method makes the queue serialize the instance, which is the
|
|
|
13
13
|
thing worth avoiding.
|
|
14
14
|
"""
|
|
15
15
|
|
|
16
|
+
import asyncio
|
|
17
|
+
import inspect
|
|
16
18
|
import logging
|
|
17
19
|
from typing import Any
|
|
18
20
|
|
|
@@ -31,6 +33,9 @@ def run_task(task_name: str, payload: str | bytes | None = None) -> Any:
|
|
|
31
33
|
``success`` runs after ``task_function`` returns; ``failure`` runs if it
|
|
32
34
|
raises, and the original exception is re-raised so the queue records the
|
|
33
35
|
job as failed. A hook that raises does not mask the original error.
|
|
36
|
+
``last_result`` and ``last_error`` are set on the task before the hook
|
|
37
|
+
runs. A coroutine ``task_function`` is awaited. ``timeout`` is not applied
|
|
38
|
+
here; the scheduling adapter owns that.
|
|
34
39
|
|
|
35
40
|
Args:
|
|
36
41
|
task_name: The registered name of the task to run.
|
|
@@ -48,10 +53,16 @@ def run_task(task_name: str, payload: str | bytes | None = None) -> Any:
|
|
|
48
53
|
|
|
49
54
|
try:
|
|
50
55
|
result = task.task_function(*args, **kwargs)
|
|
51
|
-
|
|
56
|
+
if inspect.iscoroutine(result):
|
|
57
|
+
result = asyncio.run(result)
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
task.last_result = None
|
|
60
|
+
task.last_error = exc
|
|
52
61
|
_run_hook(task, "failure", args, kwargs)
|
|
53
62
|
raise
|
|
54
63
|
|
|
64
|
+
task.last_result = result
|
|
65
|
+
task.last_error = None
|
|
55
66
|
_run_hook(task, "success", args, kwargs)
|
|
56
67
|
return result
|
|
57
68
|
|
corekit/jobs/task.py
CHANGED
|
@@ -88,6 +88,8 @@ class Task(abc.ABC):
|
|
|
88
88
|
"""
|
|
89
89
|
self.name = name or self.task_name or type(self).__name__
|
|
90
90
|
self.logger = logging.getLogger(type(self).__module__)
|
|
91
|
+
self.last_result: Any = None
|
|
92
|
+
self.last_error: BaseException | None = None
|
|
91
93
|
|
|
92
94
|
@abc.abstractmethod
|
|
93
95
|
def task_function(self, *args: Any, **kwargs: Any) -> None:
|
|
@@ -99,12 +101,18 @@ class Task(abc.ABC):
|
|
|
99
101
|
def success(self, *args: Any, **kwargs: Any) -> None:
|
|
100
102
|
"""
|
|
101
103
|
Called after ``task_function`` returns. Override to add behaviour.
|
|
104
|
+
|
|
105
|
+
``last_result`` is set before this runs. The call signature stays
|
|
106
|
+
``(*args, **kwargs)`` so an existing override is not broken.
|
|
102
107
|
"""
|
|
103
108
|
self.logger.info(f"Task {self.name} completed successfully")
|
|
104
109
|
|
|
105
110
|
def failure(self, *args: Any, **kwargs: Any) -> None:
|
|
106
111
|
"""
|
|
107
112
|
Called when ``task_function`` raises. Override to add behaviour.
|
|
113
|
+
|
|
114
|
+
``last_error`` is set before this runs. The call signature stays
|
|
115
|
+
``(*args, **kwargs)`` so an existing override is not broken.
|
|
108
116
|
"""
|
|
109
117
|
self.logger.error(f"Task {self.name} failed")
|
|
110
118
|
|
|
@@ -119,8 +127,11 @@ class ScheduledTask(Task):
|
|
|
119
127
|
"""
|
|
120
128
|
A task that runs on a schedule.
|
|
121
129
|
|
|
122
|
-
|
|
123
|
-
|
|
130
|
+
``interval``, ``repeat``, ``timeout``, and ``schedule_delay`` are for the
|
|
131
|
+
adapter that talks to a scheduler. ``run_task`` does not apply ``timeout``
|
|
132
|
+
and does not sleep for ``interval`` — wrapping a still-running queue job
|
|
133
|
+
would change production workers. Copy ``schedule_delay`` per instance so
|
|
134
|
+
one task cannot mutate the shared class dict.
|
|
124
135
|
"""
|
|
125
136
|
|
|
126
137
|
interval: int | None = None
|
|
@@ -133,6 +144,16 @@ class ScheduledTask(Task):
|
|
|
133
144
|
args: list[Any] | None = None
|
|
134
145
|
kwargs: dict[str, Any] | None = None
|
|
135
146
|
|
|
147
|
+
def __init__(self, name: str | None = None) -> None:
|
|
148
|
+
super().__init__(name)
|
|
149
|
+
# Class attributes are shared. Replace them on the instance so a later
|
|
150
|
+
# in-place edit cannot change every other scheduled task.
|
|
151
|
+
self.schedule_delay = dict(self.schedule_delay)
|
|
152
|
+
if self.args is not None:
|
|
153
|
+
self.args = list(self.args)
|
|
154
|
+
if self.kwargs is not None:
|
|
155
|
+
self.kwargs = dict(self.kwargs)
|
|
156
|
+
|
|
136
157
|
def get_start_time(self) -> datetime:
|
|
137
158
|
"""
|
|
138
159
|
When the first run should happen.
|
corekit/log_monitor/models.py
CHANGED
|
@@ -75,10 +75,16 @@ class PauseContainerAction(BaseAction):
|
|
|
75
75
|
|
|
76
76
|
|
|
77
77
|
class ExecuteCommandAction(BaseAction):
|
|
78
|
-
"""
|
|
78
|
+
"""
|
|
79
|
+
Run a command as an argument list.
|
|
80
|
+
|
|
81
|
+
``command`` is split like a shell command line, then ``{container}`` is
|
|
82
|
+
filled in. It is not passed to a shell, so pipes and redirects are not
|
|
83
|
+
available.
|
|
84
|
+
"""
|
|
79
85
|
|
|
80
86
|
type: Literal[ActionType.EXECUTE_COMMAND] = ActionType.EXECUTE_COMMAND
|
|
81
|
-
command: str = Field(..., description="
|
|
87
|
+
command: str = Field(..., description="Argument list template with a {container} placeholder")
|
|
82
88
|
|
|
83
89
|
|
|
84
90
|
# Union type for all actions
|