python-corekit 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/api/application.py +47 -9
- corekit/api/lifespan.py +26 -3
- corekit/concurrency/__init__.py +2 -2
- corekit/concurrency/decorators.py +32 -5
- corekit/concurrency/thread_local.py +2 -2
- corekit/concurrency/worker.py +9 -0
- corekit/config/loader.py +42 -5
- corekit/config/settings.py +11 -1
- corekit/connections/__init__.py +7 -1
- corekit/connections/connectable.py +45 -4
- corekit/connections/redis/connection.py +53 -10
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +39 -5
- corekit/connections/sql/fields/__init__.py +2 -2
- corekit/connections/sql/fields/jsonb.py +13 -6
- corekit/connections/sql/migration/__init__.py +4 -0
- corekit/connections/sql/migration/operations.py +69 -2
- corekit/connections/sql/operations/base.py +11 -2
- corekit/connections/sql/operations/statements.py +25 -5
- corekit/connections/sql/table.py +7 -29
- corekit/crypto/__init__.py +3 -1
- corekit/crypto/constants.py +2 -2
- corekit/crypto/hasher.py +9 -4
- corekit/data/dataset.py +8 -2
- corekit/data/expressions/__init__.py +3 -3
- corekit/data/expressions/comparison.py +19 -80
- corekit/data/expressions/expression.py +0 -32
- corekit/data/expressions/operator.py +13 -28
- corekit/data/stats.py +3 -0
- corekit/decorators/exception_handling.py +36 -8
- corekit/docker/watchdog.py +50 -31
- corekit/etl/__init__.py +2 -1
- corekit/etl/connection.py +14 -12
- corekit/etl/extract/extractor.py +6 -13
- corekit/etl/orchestrator.py +19 -2
- corekit/etl/schemas.py +2 -2
- corekit/etl/transform/transformer.py +4 -1
- corekit/events/publisher.py +1 -1
- corekit/events/reader.py +26 -21
- corekit/events/sse.py +4 -1
- corekit/events/websocket.py +24 -11
- corekit/exceptions/__init__.py +24 -9
- corekit/exceptions/base.py +139 -10
- corekit/exceptions/enum.py +17 -0
- corekit/exceptions/types.py +6 -6
- corekit/files/__init__.py +2 -4
- corekit/files/base.py +15 -2
- corekit/files/enum.py +0 -5
- corekit/files/json.py +16 -2
- corekit/http/__init__.py +43 -5
- corekit/http/api.py +24 -0
- corekit/http/client.py +100 -73
- corekit/http/exceptions.py +140 -0
- corekit/http/response.py +50 -1
- corekit/http/status.py +89 -0
- corekit/jobs/runner.py +12 -1
- corekit/jobs/task.py +23 -2
- corekit/log_monitor/models.py +8 -2
- corekit/log_monitor/service.py +77 -38
- corekit/notifications/base.py +18 -10
- corekit/observability/__init__.py +9 -2
- corekit/observability/benchmarkable.py +23 -5
- corekit/observability/loggable.py +21 -0
- corekit/observability/request_context.py +55 -2
- corekit/observability/timing/timer.py +4 -2
- corekit/registry/__init__.py +2 -2
- corekit/registry/registry.py +55 -14
- corekit/schemas/enum.py +22 -1
- corekit/schemas/types.py +6 -1
- corekit/serialization/__init__.py +2 -0
- corekit/serialization/pickle_file.py +61 -0
- corekit/serialization/serializable.py +22 -2
- corekit/serialization/serializer.py +9 -2
- corekit/utils/collections.py +22 -13
- corekit/utils/payload.py +12 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/METADATA +7 -7
- python_corekit-0.3.0.dist-info/RECORD +145 -0
- corekit/constants.py +0 -45
- corekit/exceptions/http/exceptions.py +0 -37
- corekit/files/pickle.py +0 -12
- python_corekit-0.2.0.dist-info/RECORD +0 -143
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/WHEEL +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.3.0.dist-info}/top_level.txt +0 -0
corekit/api/application.py
CHANGED
|
@@ -26,16 +26,34 @@ from typing import Any, Sequence
|
|
|
26
26
|
|
|
27
27
|
from fastapi import FastAPI
|
|
28
28
|
from fastapi.middleware.cors import CORSMiddleware
|
|
29
|
+
from fastapi.responses import JSONResponse
|
|
29
30
|
from fastapi.staticfiles import StaticFiles
|
|
30
31
|
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
|
31
32
|
|
|
32
33
|
from corekit.api.middleware import MiddlewareStack
|
|
33
34
|
from corekit.api.routers import CatchAllRouter, SimpleRouter, router_registry
|
|
35
|
+
from corekit.exceptions import CoreHTTPException
|
|
34
36
|
from corekit.observability.loggable import Loggable
|
|
35
37
|
|
|
36
38
|
__all__ = ["Application"]
|
|
37
39
|
|
|
38
40
|
|
|
41
|
+
def _origin_in_package(router: SimpleRouter, package_name: str) -> bool:
|
|
42
|
+
"""
|
|
43
|
+
Whether this router was constructed by ``package_name`` or a submodule.
|
|
44
|
+
"""
|
|
45
|
+
origin = router._origin_module
|
|
46
|
+
return origin == package_name or origin.startswith(f"{package_name}.")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
async def _core_http_exception_handler(request: Any, exc: CoreHTTPException) -> JSONResponse:
|
|
50
|
+
"""
|
|
51
|
+
The same body FastAPI returns for ``HTTPException``.
|
|
52
|
+
"""
|
|
53
|
+
headers = getattr(exc, "headers", None)
|
|
54
|
+
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}, headers=headers)
|
|
55
|
+
|
|
56
|
+
|
|
39
57
|
class Application(FastAPI, Loggable):
|
|
40
58
|
"""
|
|
41
59
|
``FastAPI`` plus router discovery, middleware helpers and static mounting.
|
|
@@ -47,6 +65,7 @@ class Application(FastAPI, Loggable):
|
|
|
47
65
|
"""
|
|
48
66
|
FastAPI.__init__(self, *args, **kwargs)
|
|
49
67
|
Loggable.__init__(self)
|
|
68
|
+
self._register_http_exception_handler()
|
|
50
69
|
|
|
51
70
|
# ─── Routers ──────────────────────────────────────────────────────────
|
|
52
71
|
|
|
@@ -55,9 +74,12 @@ class Application(FastAPI, Loggable):
|
|
|
55
74
|
Import every module under ``package`` and mount the routers it defines.
|
|
56
75
|
|
|
57
76
|
Importing a module is what constructs its routers, so discovery is a
|
|
58
|
-
walk over ``package`` that lets each one register itself.
|
|
59
|
-
|
|
60
|
-
|
|
77
|
+
walk over ``package`` that lets each one register itself. Only routers
|
|
78
|
+
constructed inside that package are mounted: the registry is
|
|
79
|
+
process-wide, and an unclaimed router from some other import must not
|
|
80
|
+
ride along. A router that was included into another router is skipped
|
|
81
|
+
-- its parent already owns it -- and any ``CatchAllRouter`` is mounted
|
|
82
|
+
last.
|
|
61
83
|
|
|
62
84
|
Args:
|
|
63
85
|
package: The imported package to search, e.g. ``app.backend.routers``.
|
|
@@ -78,11 +100,12 @@ class Application(FastAPI, Loggable):
|
|
|
78
100
|
|
|
79
101
|
self._warn_on_unimportable_dirs(package)
|
|
80
102
|
|
|
81
|
-
before = len(router_registry)
|
|
82
103
|
scanned = self._import_submodules(package)
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
104
|
+
# unclaimed is every top-level router in the process. Mounting that
|
|
105
|
+
# list would include routers built by some other package that happens
|
|
106
|
+
# to be imported. Origin is where the instance was constructed, which
|
|
107
|
+
# is the scanned package for a router that belongs here.
|
|
108
|
+
mountable = [router for router in router_registry.unclaimed if _origin_in_package(router, name)]
|
|
86
109
|
if not mountable:
|
|
87
110
|
raise ValueError(
|
|
88
111
|
f"No routers found in '{name}' (scanned {scanned} modules). "
|
|
@@ -90,8 +113,9 @@ class Application(FastAPI, Loggable):
|
|
|
90
113
|
f"a SimpleRouter or SmartRouter at import time."
|
|
91
114
|
)
|
|
92
115
|
|
|
93
|
-
|
|
94
|
-
|
|
116
|
+
in_package = [router for router in router_registry if _origin_in_package(router, name)]
|
|
117
|
+
claimed = len(in_package) - len(mountable)
|
|
118
|
+
self.info(f" scanned {scanned} modules, found {len(in_package)} routers")
|
|
95
119
|
if claimed > 0:
|
|
96
120
|
plural = "s" if claimed != 1 else ""
|
|
97
121
|
self.info(f" skipped {claimed} sub-router{plural} (already included by a parent)")
|
|
@@ -235,3 +259,17 @@ class Application(FastAPI, Loggable):
|
|
|
235
259
|
self.mount(url_path, StaticFiles(directory=str(path), **kwargs), name=name or url_path.strip("/"))
|
|
236
260
|
self.info(f" + static {url_path:<24} -> {path}")
|
|
237
261
|
return True
|
|
262
|
+
|
|
263
|
+
def _register_http_exception_handler(self) -> None:
|
|
264
|
+
"""
|
|
265
|
+
Handle the library's HTTP exceptions the same way FastAPI handles its own.
|
|
266
|
+
|
|
267
|
+
``CoreHTTPException`` is already an ``HTTPException``, so Starlette
|
|
268
|
+
would answer it. Registering the type explicitly keeps that response
|
|
269
|
+
-- ``{"detail": ...}`` only, never ``message`` or ``error`` -- even
|
|
270
|
+
if a later change in lookup order would otherwise miss the subclass.
|
|
271
|
+
A handler the caller already installed is left in place.
|
|
272
|
+
"""
|
|
273
|
+
if CoreHTTPException in self.exception_handlers:
|
|
274
|
+
return
|
|
275
|
+
self.add_exception_handler(CoreHTTPException, _core_http_exception_handler)
|
corekit/api/lifespan.py
CHANGED
|
@@ -108,7 +108,9 @@ class Lifespan(Loggable):
|
|
|
108
108
|
startup: Called on the way up, in the order steps were added.
|
|
109
109
|
shutdown: Called on the way down, in reverse order. Called only if
|
|
110
110
|
``startup`` completed, so it can assume what ``startup`` builds.
|
|
111
|
-
tags:
|
|
111
|
+
tags: Labels used by ``for_tags``. Untagged steps always run; a
|
|
112
|
+
tagged step runs only when one of its tags is selected. With
|
|
113
|
+
no filter, every step runs.
|
|
112
114
|
always_shutdown: Call ``shutdown`` even when ``startup`` raised.
|
|
113
115
|
For a startup that acquires something before it can fail -- a
|
|
114
116
|
connection opened, then a handshake that throws. The hook then
|
|
@@ -136,6 +138,25 @@ class Lifespan(Loggable):
|
|
|
136
138
|
"""The registered steps, in the order they will start."""
|
|
137
139
|
return list(self._steps)
|
|
138
140
|
|
|
141
|
+
def for_tags(self, *tags: str) -> "Lifespan":
|
|
142
|
+
"""
|
|
143
|
+
A lifespan that runs the steps matching ``tags``, plus untagged ones.
|
|
144
|
+
|
|
145
|
+
An untagged step is ordinary startup and always runs. A tagged step
|
|
146
|
+
runs only when the caller asked for one of its tags, so a label is a
|
|
147
|
+
filter rather than decoration. No tags means every step, which is
|
|
148
|
+
what ``__call__`` does.
|
|
149
|
+
"""
|
|
150
|
+
if not tags:
|
|
151
|
+
return self
|
|
152
|
+
|
|
153
|
+
wanted = set(tags)
|
|
154
|
+
selected = Lifespan()
|
|
155
|
+
for step in self._steps:
|
|
156
|
+
if not step.tags or wanted.intersection(step.tags):
|
|
157
|
+
selected._steps.append(step)
|
|
158
|
+
return selected
|
|
159
|
+
|
|
139
160
|
@asynccontextmanager
|
|
140
161
|
async def __call__(self, app: Any = None) -> AsyncIterator[None]:
|
|
141
162
|
"""
|
|
@@ -166,7 +187,8 @@ class Lifespan(Loggable):
|
|
|
166
187
|
stack.push_async_callback(self._run_shutdown, step)
|
|
167
188
|
|
|
168
189
|
if step.startup is not None:
|
|
169
|
-
|
|
190
|
+
label = step.name if not step.tags else f"{step.name} [{', '.join(step.tags)}]"
|
|
191
|
+
self.info(f" ^ {label}")
|
|
170
192
|
try:
|
|
171
193
|
await self._maybe_await(step.startup)
|
|
172
194
|
except Exception:
|
|
@@ -192,7 +214,8 @@ class Lifespan(Loggable):
|
|
|
192
214
|
self.warning(f" - {step.name} (skipped: startup did not complete)")
|
|
193
215
|
return
|
|
194
216
|
|
|
195
|
-
|
|
217
|
+
label = step.name if not step.tags else f"{step.name} [{', '.join(step.tags)}]"
|
|
218
|
+
self.info(f" v {label}")
|
|
196
219
|
try:
|
|
197
220
|
await self._maybe_await(step.shutdown)
|
|
198
221
|
except Exception:
|
corekit/concurrency/__init__.py
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
Concurrency primitives: per-thread storage, workers, and parallel mapping.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from corekit.concurrency.decorators import parallelize
|
|
5
|
+
from corekit.concurrency.decorators import allow_sync, parallelize
|
|
6
6
|
from corekit.concurrency.thread_local import ThreadLocalRegistry
|
|
7
7
|
from corekit.concurrency.worker import ThreadWorker
|
|
8
8
|
|
|
9
|
-
__all__ = ["ThreadLocalRegistry", "ThreadWorker", "parallelize"]
|
|
9
|
+
__all__ = ["ThreadLocalRegistry", "ThreadWorker", "allow_sync", "parallelize"]
|
|
@@ -18,19 +18,26 @@ machine rather than an error message.
|
|
|
18
18
|
|
|
19
19
|
Results arrive out of order, because they arrive as they complete. Pass
|
|
20
20
|
``ordered=True`` to get them in the order the inputs were given.
|
|
21
|
+
|
|
22
|
+
``allow_sync`` wraps an async function so a script can call it directly, while
|
|
23
|
+
an already-running event loop still gets a coroutine to await.
|
|
21
24
|
"""
|
|
22
25
|
|
|
26
|
+
import asyncio
|
|
23
27
|
import functools
|
|
28
|
+
import logging
|
|
29
|
+
from collections.abc import Awaitable, Callable
|
|
24
30
|
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
|
|
25
|
-
from typing import Any,
|
|
31
|
+
from typing import Any, Iterable, Iterator, ParamSpec, TypeVar
|
|
26
32
|
|
|
27
33
|
from corekit.config import get_settings
|
|
28
|
-
from corekit.observability import Loggable
|
|
29
34
|
|
|
30
|
-
__all__ = ["parallelize"]
|
|
35
|
+
__all__ = ["parallelize", "allow_sync"]
|
|
36
|
+
|
|
37
|
+
_logger = logging.getLogger("parallelize")
|
|
31
38
|
|
|
32
|
-
|
|
33
|
-
|
|
39
|
+
P = ParamSpec("P")
|
|
40
|
+
R = TypeVar("R")
|
|
34
41
|
|
|
35
42
|
|
|
36
43
|
def parallelize(
|
|
@@ -70,3 +77,23 @@ def parallelize(
|
|
|
70
77
|
return wrapper
|
|
71
78
|
|
|
72
79
|
return decorator
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def allow_sync(async_func: Callable[P, Awaitable[R]]) -> Callable[P, R | Awaitable[R]]:
|
|
83
|
+
"""
|
|
84
|
+
Call an async function from sync code, or return its coroutine if a loop is running.
|
|
85
|
+
|
|
86
|
+
A script gets the result back immediately. Inside an async app the wrapper
|
|
87
|
+
returns the coroutine, so the caller must await it — calling the wrapper
|
|
88
|
+
and ignoring the result will not run the function.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
@functools.wraps(async_func)
|
|
92
|
+
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | Awaitable[R]:
|
|
93
|
+
try:
|
|
94
|
+
asyncio.get_running_loop()
|
|
95
|
+
except RuntimeError:
|
|
96
|
+
return asyncio.run(async_func(*args, **kwargs))
|
|
97
|
+
return async_func(*args, **kwargs)
|
|
98
|
+
|
|
99
|
+
return wrapper
|
|
@@ -19,7 +19,7 @@ import threading
|
|
|
19
19
|
from typing import Any
|
|
20
20
|
|
|
21
21
|
from corekit.observability.loggable import Loggable
|
|
22
|
-
from corekit.registry import
|
|
22
|
+
from corekit.registry import normalize_key
|
|
23
23
|
|
|
24
24
|
__all__ = ["ThreadLocalRegistry"]
|
|
25
25
|
|
|
@@ -59,7 +59,7 @@ class ThreadLocalRegistry(threading.local, Loggable):
|
|
|
59
59
|
turn SQLConnection into "sqlconnection" rather than "sql_connection".
|
|
60
60
|
"""
|
|
61
61
|
name = key.__name__ if isinstance(key, type) else str(key)
|
|
62
|
-
return
|
|
62
|
+
return normalize_key(name).replace("-", "_")
|
|
63
63
|
|
|
64
64
|
def get(self, key: Any, fallback: Any = None) -> Any:
|
|
65
65
|
"""
|
corekit/concurrency/worker.py
CHANGED
|
@@ -5,6 +5,15 @@ import time
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class ThreadWorker(threading.Thread):
|
|
8
|
+
"""
|
|
9
|
+
A daemon thread that waits for ``can_start``, then calls ``process`` once.
|
|
10
|
+
|
|
11
|
+
``timeout`` and ``loop`` are stored for a subclass that wants them.
|
|
12
|
+
``run`` does not enforce ``timeout`` and does not drive ``loop``.
|
|
13
|
+
``stop`` sets ``running`` so a subclass of ``process`` can notice it; it
|
|
14
|
+
does not interrupt a thread already inside ``process``.
|
|
15
|
+
"""
|
|
16
|
+
|
|
8
17
|
def __init__(
|
|
9
18
|
self,
|
|
10
19
|
can_start: threading.Event,
|
corekit/config/loader.py
CHANGED
|
@@ -9,6 +9,8 @@ settings know only their own shape.
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import Any
|
|
11
11
|
|
|
12
|
+
from pydantic import ValidationError
|
|
13
|
+
|
|
12
14
|
from corekit.config.settings import ENV_PREFIX, CorekitSettings
|
|
13
15
|
from corekit.config.sources import ConfigFileSource, ConfigSource, EnvironmentSource, PyprojectSource
|
|
14
16
|
from corekit.observability import Loggable
|
|
@@ -72,11 +74,12 @@ class SettingsLoader(Loggable):
|
|
|
72
74
|
"""
|
|
73
75
|
Read every source and construct the settings.
|
|
74
76
|
|
|
75
|
-
Invalid configuration
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
Invalid configuration drops the bad section and keeps the rest.
|
|
78
|
+
A bad thread count must not discard a valid salt or serialization
|
|
79
|
+
key that arrived from another section or the environment. If what
|
|
80
|
+
remains still will not validate, the last resort is defaults --
|
|
81
|
+
corekit has to remain importable. The log line is the difference
|
|
82
|
+
between this and silently ignoring it.
|
|
80
83
|
"""
|
|
81
84
|
values: dict[str, Any] = {}
|
|
82
85
|
for source in self.sources():
|
|
@@ -84,10 +87,44 @@ class SettingsLoader(Loggable):
|
|
|
84
87
|
|
|
85
88
|
try:
|
|
86
89
|
return CorekitSettings(**values)
|
|
90
|
+
except ValidationError as exc:
|
|
91
|
+
cleaned, dropped = self._without_invalid_sections(values, exc)
|
|
92
|
+
if dropped:
|
|
93
|
+
self.error(
|
|
94
|
+
f"Invalid corekit configuration in {', '.join(dropped)}; "
|
|
95
|
+
f"those sections fall back to defaults: {exc}"
|
|
96
|
+
)
|
|
97
|
+
try:
|
|
98
|
+
return CorekitSettings(**cleaned)
|
|
99
|
+
except ValidationError as retry_exc:
|
|
100
|
+
self.error(f"Invalid corekit configuration, falling back to defaults: {retry_exc}")
|
|
101
|
+
return CorekitSettings()
|
|
102
|
+
self.error(f"Invalid corekit configuration, falling back to defaults: {exc}")
|
|
103
|
+
return CorekitSettings()
|
|
87
104
|
except Exception as exc:
|
|
88
105
|
self.error(f"Invalid corekit configuration, falling back to defaults: {exc}")
|
|
89
106
|
return CorekitSettings()
|
|
90
107
|
|
|
108
|
+
@staticmethod
|
|
109
|
+
def _without_invalid_sections(values: dict[str, Any], exc: ValidationError) -> tuple[dict[str, Any], list[str]]:
|
|
110
|
+
"""
|
|
111
|
+
Drop each section named in a validation error, leaving the others.
|
|
112
|
+
|
|
113
|
+
A section is the first element of the error location. A failure that
|
|
114
|
+
names no section cannot be isolated, so nothing is dropped and the
|
|
115
|
+
caller falls back to defaults.
|
|
116
|
+
"""
|
|
117
|
+
dropped: list[str] = []
|
|
118
|
+
for error in exc.errors():
|
|
119
|
+
loc = error.get("loc") or ()
|
|
120
|
+
if not loc:
|
|
121
|
+
continue
|
|
122
|
+
section = str(loc[0])
|
|
123
|
+
if section not in dropped:
|
|
124
|
+
dropped.append(section)
|
|
125
|
+
cleaned = {key: value for key, value in values.items() if key not in dropped}
|
|
126
|
+
return cleaned, dropped
|
|
127
|
+
|
|
91
128
|
@staticmethod
|
|
92
129
|
def _merge(base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
|
|
93
130
|
"""
|
corekit/config/settings.py
CHANGED
|
@@ -35,6 +35,10 @@ class StandardsSettings(BaseModel):
|
|
|
35
35
|
|
|
36
36
|
Off by default. These are conventions, not library invariants, and crashing
|
|
37
37
|
a consumer's application over a missing docstring would be hostile.
|
|
38
|
+
|
|
39
|
+
``strict_mode`` makes a ``SmartRegistry`` refuse a second write to a
|
|
40
|
+
normalized key instead of warning and replacing. It does not imply
|
|
41
|
+
``require_handler_docstrings``; that check stays its own flag.
|
|
38
42
|
"""
|
|
39
43
|
|
|
40
44
|
require_handler_docstrings: bool = False
|
|
@@ -80,7 +84,13 @@ class ConcurrencySettings(BaseModel):
|
|
|
80
84
|
@staticmethod
|
|
81
85
|
def cpu_default() -> int:
|
|
82
86
|
"""
|
|
83
|
-
A
|
|
87
|
+
A machine-sized thread count, for a caller that asks for one.
|
|
88
|
+
|
|
89
|
+
Not what ``resolve`` or ``parallelize`` use when the caller is silent.
|
|
90
|
+
Those use ``default_threads``. Pass this in when the work should
|
|
91
|
+
scale with the machine::
|
|
92
|
+
|
|
93
|
+
parallelize(num_threads=ConcurrencySettings.cpu_default())
|
|
84
94
|
"""
|
|
85
95
|
return min(32, (os.cpu_count() or 1) * 5)
|
|
86
96
|
|
corekit/connections/__init__.py
CHANGED
|
@@ -17,7 +17,12 @@ implementations of the abstraction directly above them, and there is no useful
|
|
|
17
17
|
way to think about one without the other.
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
|
-
from corekit.connections.connectable import
|
|
20
|
+
from corekit.connections.connectable import (
|
|
21
|
+
Connectable,
|
|
22
|
+
ConnectableType,
|
|
23
|
+
ConnectionPreference,
|
|
24
|
+
ReadOnlyConnectionError,
|
|
25
|
+
)
|
|
21
26
|
from corekit.connections.decorators import connect
|
|
22
27
|
from corekit.connections.registry import ConnectionRegistry, registry
|
|
23
28
|
|
|
@@ -26,6 +31,7 @@ __all__ = [
|
|
|
26
31
|
"ConnectableType",
|
|
27
32
|
"ConnectionPreference",
|
|
28
33
|
"ConnectionRegistry",
|
|
34
|
+
"ReadOnlyConnectionError",
|
|
29
35
|
"connect",
|
|
30
36
|
"registry",
|
|
31
37
|
]
|
|
@@ -21,22 +21,40 @@ class without importing it directly.
|
|
|
21
21
|
from abc import ABC, abstractmethod
|
|
22
22
|
from typing import Any
|
|
23
23
|
|
|
24
|
+
from corekit.exceptions import InternalCoreException, Retryability
|
|
24
25
|
from corekit.observability.loggable import Loggable
|
|
25
26
|
from corekit.registry import SmartRegistry
|
|
26
27
|
from corekit.schemas.enum import StringEnum
|
|
27
28
|
|
|
28
|
-
__all__ = ["Connectable", "ConnectableType", "ConnectionPreference"]
|
|
29
|
+
__all__ = ["Connectable", "ConnectableType", "ConnectionPreference", "ReadOnlyConnectionError"]
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
class ConnectionPreference(StringEnum):
|
|
32
33
|
"""
|
|
33
|
-
|
|
34
|
+
A stored hint for which path a caller prefers.
|
|
35
|
+
|
|
36
|
+
The base class records this and does not consult it. ``connect`` and
|
|
37
|
+
``async_connect`` follow the method the caller invoked, and ``@connect``
|
|
38
|
+
always opens with ``with``. A subclass may read the hint; nothing here
|
|
39
|
+
switches paths because of it.
|
|
34
40
|
"""
|
|
35
41
|
|
|
36
42
|
ASYNC = "async"
|
|
37
43
|
SYNC = "sync"
|
|
38
44
|
|
|
39
45
|
|
|
46
|
+
class ReadOnlyConnectionError(InternalCoreException):
|
|
47
|
+
"""
|
|
48
|
+
Raised when a write is attempted on a connection opened read-only.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(self, connection: str) -> None:
|
|
52
|
+
super().__init__(
|
|
53
|
+
message=f"{connection} is read-only",
|
|
54
|
+
retryable=Retryability.NON_RETRYABLE,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
40
58
|
class Connectable(Loggable, ABC):
|
|
41
59
|
"""
|
|
42
60
|
Base class for anything with a connection lifecycle.
|
|
@@ -97,10 +115,20 @@ class Connectable(Loggable, ABC):
|
|
|
97
115
|
@property
|
|
98
116
|
def connection_preference(self) -> ConnectionPreference:
|
|
99
117
|
"""
|
|
100
|
-
|
|
118
|
+
The path hint recorded at construction. Not used to open the connection.
|
|
101
119
|
"""
|
|
102
120
|
return self._connection_preference
|
|
103
121
|
|
|
122
|
+
def require_writable(self) -> None:
|
|
123
|
+
"""
|
|
124
|
+
Raise if this connection was opened for reading only.
|
|
125
|
+
|
|
126
|
+
Corekit's own write methods call this. The base class does not wrap
|
|
127
|
+
every method, so a subclass with its own writes should call it too.
|
|
128
|
+
"""
|
|
129
|
+
if self._read_only:
|
|
130
|
+
raise ReadOnlyConnectionError(type(self).__name__)
|
|
131
|
+
|
|
104
132
|
def __repr__(self) -> str:
|
|
105
133
|
return f"{self.__class__.__name__}(connected={self.is_connected})"
|
|
106
134
|
|
|
@@ -190,11 +218,24 @@ class Connectable(Loggable, ABC):
|
|
|
190
218
|
self.debug(f"Terminating {self.__class__.__name__} connection")
|
|
191
219
|
self._disconnect()
|
|
192
220
|
|
|
221
|
+
def _async_is_open(self) -> bool:
|
|
222
|
+
"""
|
|
223
|
+
Whether the async path is already up.
|
|
224
|
+
|
|
225
|
+
``is_connected`` is the sync flag. Redis keeps a process-wide sync pool
|
|
226
|
+
and a separate async client, so a sync pool must not stand in for the
|
|
227
|
+
async client. Classes without ``is_async_connected`` use ``is_connected``.
|
|
228
|
+
"""
|
|
229
|
+
for cls in type(self).mro():
|
|
230
|
+
if "is_async_connected" in cls.__dict__:
|
|
231
|
+
return bool(self.is_async_connected)
|
|
232
|
+
return self.is_connected
|
|
233
|
+
|
|
193
234
|
async def async_connect(self, force_reconnect: bool = False) -> None:
|
|
194
235
|
"""
|
|
195
236
|
Connect asynchronously if not already connected.
|
|
196
237
|
"""
|
|
197
|
-
if self.
|
|
238
|
+
if self._async_is_open() and not force_reconnect:
|
|
198
239
|
return
|
|
199
240
|
|
|
200
241
|
self.debug(f"Establishing async {self.__class__.__name__} connection")
|
|
@@ -23,6 +23,7 @@ from redis import asyncio as aioredis
|
|
|
23
23
|
|
|
24
24
|
from corekit.config import get_settings
|
|
25
25
|
from corekit.connections import Connectable
|
|
26
|
+
from corekit.exceptions import InternalCoreException, Retryability
|
|
26
27
|
|
|
27
28
|
__all__ = ["RedisConnection", "RedisNotConnectedError"]
|
|
28
29
|
|
|
@@ -31,11 +32,14 @@ logger = logging.getLogger(__name__)
|
|
|
31
32
|
DEFAULT_URL = "redis://localhost:6379/0"
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
class RedisNotConnectedError(
|
|
35
|
+
class RedisNotConnectedError(InternalCoreException):
|
|
35
36
|
"""
|
|
36
37
|
Raised when a client is used before its connection is opened.
|
|
37
38
|
"""
|
|
38
39
|
|
|
40
|
+
def __init__(self, message: str, *, error: str | None = None) -> None:
|
|
41
|
+
super().__init__(message, retryable=Retryability.NON_RETRYABLE, error=error)
|
|
42
|
+
|
|
39
43
|
|
|
40
44
|
def _encode(value: Any) -> str:
|
|
41
45
|
"""
|
|
@@ -65,14 +69,15 @@ class RedisConnection(Connectable):
|
|
|
65
69
|
"""
|
|
66
70
|
A Connectable wrapper over a Redis client.
|
|
67
71
|
|
|
68
|
-
The sync client is shared per URL for the life of
|
|
72
|
+
The sync client is shared per URL for the life of its holders, because a
|
|
69
73
|
Redis client is a connection pool and building one per caller defeats it.
|
|
70
|
-
``
|
|
71
|
-
pool
|
|
72
|
-
bound to a running event loop.
|
|
74
|
+
Each ``connect`` takes a hold; ``disconnect`` releases it and closes the
|
|
75
|
+
pool only when no holder remains. Async clients are per-instance, since they
|
|
76
|
+
are bound to a running event loop.
|
|
73
77
|
"""
|
|
74
78
|
|
|
75
79
|
_shared_clients: dict[str, Any] = {}
|
|
80
|
+
_shared_refs: dict[str, int] = {}
|
|
76
81
|
|
|
77
82
|
def __init__(self, url: str | None = None, safe: bool = False, **kwargs: Any) -> None:
|
|
78
83
|
"""
|
|
@@ -83,6 +88,7 @@ class RedisConnection(Connectable):
|
|
|
83
88
|
self._url = url or get_settings().redis.url or DEFAULT_URL
|
|
84
89
|
self._safe = safe
|
|
85
90
|
self._async_client: Any = None
|
|
91
|
+
self._holds_shared = False
|
|
86
92
|
|
|
87
93
|
@property
|
|
88
94
|
def url(self) -> str:
|
|
@@ -93,7 +99,7 @@ class RedisConnection(Connectable):
|
|
|
93
99
|
|
|
94
100
|
@property
|
|
95
101
|
def is_connected(self) -> bool:
|
|
96
|
-
return
|
|
102
|
+
return self._holds_shared
|
|
97
103
|
|
|
98
104
|
@property
|
|
99
105
|
def is_async_connected(self) -> bool:
|
|
@@ -102,8 +108,10 @@ class RedisConnection(Connectable):
|
|
|
102
108
|
@property
|
|
103
109
|
def client(self) -> Any:
|
|
104
110
|
"""
|
|
105
|
-
The sync client, raising if
|
|
111
|
+
The sync client, raising if this instance has not connected.
|
|
106
112
|
"""
|
|
113
|
+
if not self._holds_shared:
|
|
114
|
+
raise RedisNotConnectedError("RedisConnection is not connected. Call connect() or use a with block.")
|
|
107
115
|
existing = RedisConnection._shared_clients.get(self._url)
|
|
108
116
|
if existing is None:
|
|
109
117
|
raise RedisNotConnectedError("RedisConnection is not connected. Call connect() or use a with block.")
|
|
@@ -135,11 +143,29 @@ class RedisConnection(Connectable):
|
|
|
135
143
|
if RedisConnection._shared_clients.get(self._url) is None:
|
|
136
144
|
self.info(f"Creating shared Redis client for {self._url}")
|
|
137
145
|
RedisConnection._shared_clients[self._url] = self._build(is_async=False)
|
|
146
|
+
RedisConnection._shared_refs[self._url] = 0
|
|
147
|
+
if not self._holds_shared:
|
|
148
|
+
RedisConnection._shared_refs[self._url] = RedisConnection._shared_refs.get(self._url, 0) + 1
|
|
149
|
+
self._holds_shared = True
|
|
138
150
|
|
|
139
151
|
def _disconnect(self) -> None:
|
|
140
152
|
"""
|
|
141
|
-
|
|
153
|
+
Release this instance's hold on the shared pool; close it when last.
|
|
142
154
|
"""
|
|
155
|
+
if not self._holds_shared:
|
|
156
|
+
return
|
|
157
|
+
self._holds_shared = False
|
|
158
|
+
refs = RedisConnection._shared_refs.get(self._url, 1) - 1
|
|
159
|
+
if refs <= 0:
|
|
160
|
+
client = RedisConnection._shared_clients.pop(self._url, None)
|
|
161
|
+
RedisConnection._shared_refs.pop(self._url, None)
|
|
162
|
+
if client is not None:
|
|
163
|
+
try:
|
|
164
|
+
client.close()
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
self.debug(f"Error closing Redis client for {self._url}: {exc}")
|
|
167
|
+
else:
|
|
168
|
+
RedisConnection._shared_refs[self._url] = refs
|
|
143
169
|
|
|
144
170
|
async def _async_connect(self) -> None:
|
|
145
171
|
self._async_client = self._build(is_async=True)
|
|
@@ -166,6 +192,7 @@ class RedisConnection(Connectable):
|
|
|
166
192
|
except Exception as exc:
|
|
167
193
|
logger.debug("Error closing Redis client for %s: %s", url, exc)
|
|
168
194
|
del cls._shared_clients[url]
|
|
195
|
+
cls._shared_refs.clear()
|
|
169
196
|
|
|
170
197
|
# ------------------------------------------------------------------
|
|
171
198
|
# Sync API
|
|
@@ -175,9 +202,11 @@ class RedisConnection(Connectable):
|
|
|
175
202
|
return _decode(self.client.get(key))
|
|
176
203
|
|
|
177
204
|
def set(self, key: str, value: Any, **kwargs: Any) -> None:
|
|
205
|
+
self.require_writable()
|
|
178
206
|
self.client.set(key, _encode(value), **kwargs)
|
|
179
207
|
|
|
180
208
|
def delete(self, *keys: str) -> None:
|
|
209
|
+
self.require_writable()
|
|
181
210
|
self.client.delete(*keys)
|
|
182
211
|
|
|
183
212
|
def ping(self) -> bool:
|
|
@@ -187,15 +216,18 @@ class RedisConnection(Connectable):
|
|
|
187
216
|
return bool(self.client.exists(key))
|
|
188
217
|
|
|
189
218
|
def expire(self, key: str, seconds: int) -> None:
|
|
219
|
+
self.require_writable()
|
|
190
220
|
self.client.expire(key, seconds)
|
|
191
221
|
|
|
192
222
|
def ttl(self, key: str) -> int:
|
|
193
223
|
return self.client.ttl(key)
|
|
194
224
|
|
|
195
225
|
def incr(self, key: str, amount: int = 1) -> int:
|
|
226
|
+
self.require_writable()
|
|
196
227
|
return self.client.incr(key, amount)
|
|
197
228
|
|
|
198
229
|
def decr(self, key: str, amount: int = 1) -> int:
|
|
230
|
+
self.require_writable()
|
|
199
231
|
return self.client.decr(key, amount)
|
|
200
232
|
|
|
201
233
|
def keys(self, pattern: str = "*") -> list[Any]:
|
|
@@ -205,13 +237,19 @@ class RedisConnection(Connectable):
|
|
|
205
237
|
return _decode(self.client.hget(key, field))
|
|
206
238
|
|
|
207
239
|
def hset(self, key: str, field: str, value: Any) -> None:
|
|
240
|
+
self.require_writable()
|
|
208
241
|
self.client.hset(key, field, _encode(value))
|
|
209
242
|
|
|
210
243
|
def hdel(self, key: str, *fields: str) -> None:
|
|
244
|
+
self.require_writable()
|
|
211
245
|
self.client.hdel(key, *fields)
|
|
212
246
|
|
|
213
247
|
def publish(self, channel: str, message: Any) -> int:
|
|
214
|
-
|
|
248
|
+
self.require_writable()
|
|
249
|
+
# Pre-serialized pub/sub payloads (JSON strings, bytes) pass through;
|
|
250
|
+
# structured values are encoded the same way as cache writes.
|
|
251
|
+
payload = message if isinstance(message, (str, bytes, bytearray)) else _encode(message)
|
|
252
|
+
return self.client.publish(channel, payload)
|
|
215
253
|
|
|
216
254
|
def pubsub(self) -> Any:
|
|
217
255
|
return self.client.pubsub()
|
|
@@ -224,16 +262,21 @@ class RedisConnection(Connectable):
|
|
|
224
262
|
return _decode(await self.async_client.get(key))
|
|
225
263
|
|
|
226
264
|
async def aset(self, key: str, value: Any, **kwargs: Any) -> None:
|
|
265
|
+
self.require_writable()
|
|
227
266
|
await self.async_client.set(key, _encode(value), **kwargs)
|
|
228
267
|
|
|
229
268
|
async def asetex(self, key: str, seconds: int, value: Any) -> None:
|
|
269
|
+
self.require_writable()
|
|
230
270
|
await self.async_client.setex(key, seconds, _encode(value))
|
|
231
271
|
|
|
232
272
|
async def adelete(self, *keys: str) -> None:
|
|
273
|
+
self.require_writable()
|
|
233
274
|
await self.async_client.delete(*keys)
|
|
234
275
|
|
|
235
276
|
async def akeys(self, pattern: str = "*") -> list[Any]:
|
|
236
277
|
return await self.async_client.keys(pattern)
|
|
237
278
|
|
|
238
279
|
async def apublish(self, channel: str, message: Any) -> int:
|
|
239
|
-
|
|
280
|
+
self.require_writable()
|
|
281
|
+
payload = message if isinstance(message, (str, bytes, bytearray)) else _encode(message)
|
|
282
|
+
return await self.async_client.publish(channel, payload)
|