python-corekit 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/api/application.py +47 -9
- corekit/api/lifespan.py +26 -3
- corekit/concurrency/__init__.py +2 -2
- corekit/concurrency/decorators.py +32 -5
- corekit/concurrency/thread_local.py +2 -2
- corekit/concurrency/worker.py +9 -0
- corekit/config/loader.py +42 -5
- corekit/config/settings.py +11 -1
- corekit/connections/__init__.py +7 -1
- corekit/connections/connectable.py +45 -4
- corekit/connections/redis/connection.py +53 -10
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +39 -5
- corekit/connections/sql/fields/__init__.py +2 -2
- corekit/connections/sql/fields/jsonb.py +13 -6
- corekit/connections/sql/migration/__init__.py +4 -0
- corekit/connections/sql/migration/operations.py +69 -2
- corekit/connections/sql/operations/base.py +11 -2
- corekit/connections/sql/operations/statements.py +25 -5
- corekit/connections/sql/table.py +7 -29
- corekit/crypto/__init__.py +3 -1
- corekit/crypto/constants.py +2 -2
- corekit/crypto/hasher.py +9 -4
- corekit/data/dataset.py +8 -2
- corekit/data/expressions/__init__.py +3 -3
- corekit/data/expressions/comparison.py +19 -80
- corekit/data/expressions/expression.py +0 -32
- corekit/data/expressions/operator.py +13 -28
- corekit/data/stats.py +3 -0
- corekit/decorators/exception_handling.py +36 -8
- corekit/docker/watchdog.py +50 -31
- corekit/etl/__init__.py +2 -1
- corekit/etl/connection.py +14 -12
- corekit/etl/extract/extractor.py +6 -13
- corekit/etl/orchestrator.py +19 -2
- corekit/etl/schemas.py +2 -2
- corekit/etl/transform/transformer.py +4 -1
- corekit/events/publisher.py +1 -1
- corekit/events/reader.py +26 -21
- corekit/events/sse.py +4 -1
- corekit/events/websocket.py +24 -11
- corekit/exceptions/__init__.py +24 -9
- corekit/exceptions/base.py +139 -10
- corekit/exceptions/enum.py +17 -0
- corekit/exceptions/types.py +6 -6
- corekit/files/__init__.py +2 -4
- corekit/files/base.py +15 -2
- corekit/files/enum.py +0 -5
- corekit/files/json.py +16 -2
- corekit/http/__init__.py +48 -5
- corekit/http/api.py +24 -0
- corekit/http/client.py +133 -75
- corekit/http/exceptions.py +140 -0
- corekit/http/response.py +50 -1
- corekit/http/status.py +89 -0
- corekit/http/stream.py +110 -0
- corekit/jobs/runner.py +12 -1
- corekit/jobs/task.py +23 -2
- corekit/llm/__init__.py +134 -0
- corekit/llm/client.py +179 -0
- corekit/llm/enum.py +123 -0
- corekit/llm/events.py +96 -0
- corekit/llm/messages.py +173 -0
- corekit/llm/prompts/__init__.py +19 -0
- corekit/llm/prompts/enum.py +54 -0
- corekit/llm/prompts/exceptions.py +22 -0
- corekit/llm/prompts/loader.py +139 -0
- corekit/llm/prompts/template.py +53 -0
- corekit/llm/protocols.py +65 -0
- corekit/llm/streaming.py +149 -0
- corekit/llm/tools/__init__.py +19 -0
- corekit/llm/tools/base.py +118 -0
- corekit/llm/tools/detection.py +99 -0
- corekit/llm/tools/loop.py +255 -0
- corekit/llm/tools/registry.py +103 -0
- corekit/llm/wire.py +199 -0
- corekit/log_monitor/models.py +8 -2
- corekit/log_monitor/service.py +77 -38
- corekit/notifications/base.py +18 -10
- corekit/observability/__init__.py +12 -5
- corekit/observability/benchmarkable.py +37 -5
- corekit/observability/loggable.py +21 -0
- corekit/observability/request_context.py +55 -2
- corekit/observability/timing/split.py +14 -0
- corekit/observability/timing/timer.py +33 -9
- corekit/registry/__init__.py +2 -2
- corekit/registry/registry.py +55 -14
- corekit/schemas/__init__.py +2 -1
- corekit/schemas/enum.py +22 -1
- corekit/schemas/types.py +6 -1
- corekit/schemas/version.py +58 -0
- corekit/serialization/__init__.py +2 -0
- corekit/serialization/pickle_file.py +61 -0
- corekit/serialization/serializable.py +22 -2
- corekit/serialization/serializer.py +9 -2
- corekit/utils/__init__.py +2 -1
- corekit/utils/collections.py +38 -14
- corekit/utils/payload.py +12 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/METADATA +38 -9
- python_corekit-0.4.0.dist-info/RECORD +165 -0
- corekit/constants.py +0 -45
- corekit/exceptions/http/exceptions.py +0 -37
- corekit/files/pickle.py +0 -12
- python_corekit-0.2.0.dist-info/RECORD +0 -143
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/WHEEL +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.2.0.dist-info → python_corekit-0.4.0.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,20 @@ class Split(BaseModel):
|
|
|
8
8
|
name: str | None = None
|
|
9
9
|
split_name: str | None = None
|
|
10
10
|
|
|
11
|
+
@property
|
|
12
|
+
def total_ms(self) -> float:
|
|
13
|
+
"""
|
|
14
|
+
``total`` in milliseconds.
|
|
15
|
+
"""
|
|
16
|
+
return self.total * 1000
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def latest_ms(self) -> float:
|
|
20
|
+
"""
|
|
21
|
+
``latest`` in milliseconds.
|
|
22
|
+
"""
|
|
23
|
+
return self.latest * 1000
|
|
24
|
+
|
|
11
25
|
def __str__(self) -> str:
|
|
12
26
|
identifier = ""
|
|
13
27
|
if self.name:
|
|
@@ -5,26 +5,50 @@ from corekit.observability.timing.split import Split
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class Timer:
|
|
8
|
+
start: float
|
|
9
|
+
latest: float
|
|
10
|
+
num: int
|
|
11
|
+
precision: int
|
|
12
|
+
name: str
|
|
13
|
+
|
|
8
14
|
def __init__(self, precision: int = DEFAULT_PRECISION) -> None:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
self.latest = _time
|
|
12
|
-
self.num = 0
|
|
13
|
-
self.name = type(self).__name__
|
|
15
|
+
# perf_counter is monotonic. time.time() can step backwards, which
|
|
16
|
+
# makes a split look negative for no reason the caller can act on.
|
|
14
17
|
self.precision = precision if precision > 0 else DEFAULT_PRECISION
|
|
18
|
+
self.name = type(self).__name__
|
|
19
|
+
self.reset()
|
|
20
|
+
|
|
21
|
+
def reset(self) -> None:
|
|
22
|
+
"""
|
|
23
|
+
Restart the clock and clear split numbering.
|
|
24
|
+
|
|
25
|
+
Use at the start of a new timed unit of work (a request, a model turn,
|
|
26
|
+
a pipeline stage) so ``total`` measures that unit rather than the
|
|
27
|
+
object's whole lifetime.
|
|
28
|
+
"""
|
|
29
|
+
now = time.perf_counter()
|
|
30
|
+
self.start = now
|
|
31
|
+
self.latest = now
|
|
32
|
+
self.num = 0
|
|
15
33
|
|
|
16
34
|
def _round(self, value: float) -> float:
|
|
17
35
|
return round(value, self.precision)
|
|
18
36
|
|
|
37
|
+
def elapsed(self) -> float:
|
|
38
|
+
"""
|
|
39
|
+
Seconds since start (or last ``reset``), without recording a split.
|
|
40
|
+
"""
|
|
41
|
+
return self._round(time.perf_counter() - self.start)
|
|
42
|
+
|
|
19
43
|
def split(self, split_name: str | None = None) -> Split:
|
|
20
|
-
|
|
44
|
+
now = time.perf_counter()
|
|
21
45
|
self.num += 1
|
|
22
46
|
split = Split(
|
|
23
47
|
num=self.num,
|
|
24
|
-
total=self._round(
|
|
25
|
-
latest=self._round(
|
|
48
|
+
total=self._round(now - self.start),
|
|
49
|
+
latest=self._round(now - self.latest),
|
|
26
50
|
name=self.name,
|
|
27
51
|
split_name=split_name,
|
|
28
52
|
)
|
|
29
|
-
self.latest =
|
|
53
|
+
self.latest = now
|
|
30
54
|
return split
|
corekit/registry/__init__.py
CHANGED
|
@@ -12,6 +12,6 @@ duplicates. Reach for it when position is what matters and no name is needed.
|
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
from corekit.registry.ordered import OrderedRegistry
|
|
15
|
-
from corekit.registry.registry import SmartRegistry
|
|
15
|
+
from corekit.registry.registry import SmartRegistry, normalize_key
|
|
16
16
|
|
|
17
|
-
__all__ = ["OrderedRegistry", "SmartRegistry"]
|
|
17
|
+
__all__ = ["OrderedRegistry", "SmartRegistry", "normalize_key"]
|
corekit/registry/registry.py
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import re
|
|
2
3
|
from typing import Any, Iterator
|
|
3
4
|
|
|
5
|
+
from corekit.config import get_settings
|
|
6
|
+
|
|
4
7
|
REPLACEMENT_CHAR = "-"
|
|
5
8
|
NORMALIZATION_PATTERN = re.compile(r"[\s_-]+")
|
|
6
9
|
# Split CamelCase into words: "HTTPServerError" -> "HTTP-Server-Error".
|
|
7
10
|
CAMEL_BOUNDARY_PATTERN = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
|
|
8
11
|
|
|
12
|
+
_logger = logging.getLogger("SmartRegistry")
|
|
13
|
+
_MISSING = object()
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
class SmartRegistry:
|
|
11
17
|
"""
|
|
@@ -31,19 +37,33 @@ class SmartRegistry:
|
|
|
31
37
|
"""
|
|
32
38
|
Retrieve an item from the registry using a normalized key.
|
|
33
39
|
"""
|
|
34
|
-
return self.__registry__[
|
|
40
|
+
return self.__registry__[normalize_key(key)]
|
|
35
41
|
|
|
36
42
|
def __setitem__(self, key: str, value: Any) -> None:
|
|
37
43
|
"""
|
|
38
44
|
Store an item in the registry with a normalized key.
|
|
39
|
-
|
|
40
|
-
|
|
45
|
+
|
|
46
|
+
A second write to the same normalized key used to replace the first
|
|
47
|
+
with no signal. That drops a handler that spelled the same name a
|
|
48
|
+
different way. The replacement is still allowed so an intentional
|
|
49
|
+
update works; it is warned, and refused when ``strict_mode`` is on.
|
|
50
|
+
"""
|
|
51
|
+
normalized = normalize_key(key)
|
|
52
|
+
existing = self.__registry__.get(normalized, _MISSING)
|
|
53
|
+
if existing is not _MISSING and existing is not value:
|
|
54
|
+
if _strict_mode():
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"Registry already contains {normalized!r}. "
|
|
57
|
+
f"strict_mode refuses a second registration under the same key."
|
|
58
|
+
)
|
|
59
|
+
_logger.warning(f"Overwriting registry key {normalized!r}")
|
|
60
|
+
self.__registry__[normalized] = value
|
|
41
61
|
|
|
42
62
|
def __delitem__(self, key: str) -> None:
|
|
43
63
|
"""
|
|
44
64
|
Remove an item from the registry.
|
|
45
65
|
"""
|
|
46
|
-
del self.__registry__[
|
|
66
|
+
del self.__registry__[normalize_key(key)]
|
|
47
67
|
|
|
48
68
|
def __iter__(self) -> Iterator[str]:
|
|
49
69
|
"""
|
|
@@ -73,7 +93,7 @@ class SmartRegistry:
|
|
|
73
93
|
"""
|
|
74
94
|
Check if a normalized key exists in the registry.
|
|
75
95
|
"""
|
|
76
|
-
return
|
|
96
|
+
return normalize_key(key) in self.__registry__
|
|
77
97
|
|
|
78
98
|
def __str__(self) -> str:
|
|
79
99
|
"""
|
|
@@ -114,21 +134,42 @@ class SmartRegistry:
|
|
|
114
134
|
self.__registry__ = state
|
|
115
135
|
|
|
116
136
|
@staticmethod
|
|
117
|
-
def
|
|
137
|
+
def normalize_key(key: str) -> str:
|
|
118
138
|
"""
|
|
119
139
|
Reduce a key to a canonical hyphenated form.
|
|
120
140
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
and ``"Greeting Handler"`` all normalize to ``"greeting-handler"``.
|
|
124
|
-
Without the CamelCase step a class registered under its ``__name__``
|
|
125
|
-
could not be found by the snake_case name a caller would naturally type.
|
|
141
|
+
Public so callers do not have to reach for the old private name.
|
|
142
|
+
``__normalize_key__`` remains as an alias.
|
|
126
143
|
"""
|
|
127
|
-
|
|
128
|
-
|
|
144
|
+
return normalize_key(key)
|
|
145
|
+
|
|
146
|
+
# Kept so existing callers, including ThreadLocalRegistry and tests, keep
|
|
147
|
+
# working. New code should call ``normalize_key``.
|
|
148
|
+
__normalize_key__ = staticmethod(normalize_key)
|
|
129
149
|
|
|
130
150
|
def get(self, key: str, fallback: Any = None) -> Any:
|
|
131
151
|
"""
|
|
132
152
|
Safely retrieve an item from the registry with an optional fallback.
|
|
133
153
|
"""
|
|
134
|
-
return self.__registry__.get(
|
|
154
|
+
return self.__registry__.get(normalize_key(key), fallback)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _strict_mode() -> bool:
|
|
158
|
+
"""
|
|
159
|
+
Whether a colliding key should be refused.
|
|
160
|
+
"""
|
|
161
|
+
return get_settings().standards.strict_mode
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def normalize_key(key: str) -> str:
|
|
165
|
+
"""
|
|
166
|
+
Reduce a key to a canonical hyphenated form.
|
|
167
|
+
|
|
168
|
+
Word boundaries are taken from CamelCase as well as from whitespace,
|
|
169
|
+
underscores and hyphens, so ``"GreetingHandler"``, ``"greeting_handler"``
|
|
170
|
+
and ``"Greeting Handler"`` all normalize to ``"greeting-handler"``.
|
|
171
|
+
Without the CamelCase step a class registered under its ``__name__``
|
|
172
|
+
could not be found by the snake_case name a caller would naturally type.
|
|
173
|
+
"""
|
|
174
|
+
spaced = CAMEL_BOUNDARY_PATTERN.sub(REPLACEMENT_CHAR, key.strip())
|
|
175
|
+
return NORMALIZATION_PATTERN.sub(REPLACEMENT_CHAR, spaced.lower()).strip(REPLACEMENT_CHAR)
|
corekit/schemas/__init__.py
CHANGED
|
@@ -6,5 +6,6 @@ raises where it is introduced rather than somewhere further along.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from corekit.schemas.enum import IntegerEnum, StringEnum, ValidatingEnum
|
|
9
|
+
from corekit.schemas.version import SemanticVersion
|
|
9
10
|
|
|
10
|
-
__all__ = ["IntegerEnum", "StringEnum", "ValidatingEnum"]
|
|
11
|
+
__all__ = ["IntegerEnum", "SemanticVersion", "StringEnum", "ValidatingEnum"]
|
corekit/schemas/enum.py
CHANGED
|
@@ -18,7 +18,7 @@ class ValidatingEnum(Enum):
|
|
|
18
18
|
def validate_and_create(cls, value: Any) -> "ValidatingEnum":
|
|
19
19
|
if cls.is_member(value):
|
|
20
20
|
return cls(value)
|
|
21
|
-
raise ValueError(f"{cls.__name__} does not contain {
|
|
21
|
+
raise ValueError(f"{cls.__name__} does not contain {value!r}")
|
|
22
22
|
|
|
23
23
|
@classmethod
|
|
24
24
|
def get_all_members(cls) -> list["ValidatingEnum"]:
|
|
@@ -32,6 +32,15 @@ class ValidatingEnum(Enum):
|
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
class StringEnum(str, ValidatingEnum):
|
|
35
|
+
"""
|
|
36
|
+
String-valued enum that stringifies to its value.
|
|
37
|
+
|
|
38
|
+
Plain ``(str, Enum)`` members compare equal to their value but ``str()`` /
|
|
39
|
+
f-strings still render as ``ClassName.MEMBER``. Override that so DB defaults,
|
|
40
|
+
status comparisons, and log lines get ``"pending"`` rather than
|
|
41
|
+
``"AccountRequestStatus.PENDING"``.
|
|
42
|
+
"""
|
|
43
|
+
|
|
35
44
|
@classmethod
|
|
36
45
|
def from_string(cls, value: str) -> "StringEnum":
|
|
37
46
|
"""
|
|
@@ -39,6 +48,12 @@ class StringEnum(str, ValidatingEnum):
|
|
|
39
48
|
"""
|
|
40
49
|
return cls.validate_and_create(value)
|
|
41
50
|
|
|
51
|
+
def __str__(self) -> str:
|
|
52
|
+
return str(self.value)
|
|
53
|
+
|
|
54
|
+
def __format__(self, format_spec: str) -> str:
|
|
55
|
+
return self.value.__format__(format_spec)
|
|
56
|
+
|
|
42
57
|
|
|
43
58
|
class IntegerEnum(int, ValidatingEnum):
|
|
44
59
|
@classmethod
|
|
@@ -47,3 +62,9 @@ class IntegerEnum(int, ValidatingEnum):
|
|
|
47
62
|
Convert an integer to the corresponding enum member.
|
|
48
63
|
"""
|
|
49
64
|
return cls.validate_and_create(value)
|
|
65
|
+
|
|
66
|
+
def __str__(self) -> str:
|
|
67
|
+
return str(self.value)
|
|
68
|
+
|
|
69
|
+
def __format__(self, format_spec: str) -> str:
|
|
70
|
+
return self.value.__format__(format_spec)
|
corekit/schemas/types.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from datetime import date, datetime
|
|
2
|
-
from typing import Any, Sequence
|
|
2
|
+
from typing import Any, Awaitable, Callable, Coroutine, Sequence
|
|
3
3
|
|
|
4
4
|
# ========== Primitive Types ==========
|
|
5
5
|
Number = int | float
|
|
@@ -38,3 +38,8 @@ UnknownSet = set[Any]
|
|
|
38
38
|
|
|
39
39
|
# ========== Date Types ==========
|
|
40
40
|
ArbitraryDate = date | datetime
|
|
41
|
+
|
|
42
|
+
# ========== Function Types ==========
|
|
43
|
+
AsyncFunction = Callable[..., Awaitable[Any] | Coroutine[Any, Any, Any]]
|
|
44
|
+
SyncFunction = Callable[..., Any]
|
|
45
|
+
ArbitraryFunction = AsyncFunction | SyncFunction
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Semantic version (``major.minor.patch`` with optional ``-extra``).
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from typing import Self
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
from corekit.utils import safe_int, safe_string
|
|
13
|
+
|
|
14
|
+
__all__ = ["SemanticVersion"]
|
|
15
|
+
|
|
16
|
+
_VERSION_RE = re.compile(r"^(?P<major>\d+)(?:\.(?P<minor>\d+)(?:\.(?P<patch>\d+)?)?)?(?:-(?P<extra>[A-Za-z0-9._-]+))?$")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SemanticVersion(BaseModel):
|
|
20
|
+
"""
|
|
21
|
+
Structured ``major.minor.patch`` with an optional ``-extra`` suffix.
|
|
22
|
+
|
|
23
|
+
Suitable for filenames, config pins, and other places that need a parsed
|
|
24
|
+
SemVer-like value without tying to packaging metadata.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
major: int = Field(ge=0)
|
|
28
|
+
minor: int = Field(default=0, ge=0)
|
|
29
|
+
patch: int = Field(default=0, ge=0)
|
|
30
|
+
extra: str | None = None
|
|
31
|
+
|
|
32
|
+
def __str__(self) -> str:
|
|
33
|
+
version = f"{self.major}.{self.minor}.{self.patch}"
|
|
34
|
+
if self.extra:
|
|
35
|
+
return f"{version}-{self.extra}"
|
|
36
|
+
return version
|
|
37
|
+
|
|
38
|
+
def __repr__(self) -> str:
|
|
39
|
+
return f"SemanticVersion({self})"
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def parse(cls, value: SemanticVersion | str | int) -> Self:
|
|
43
|
+
"""
|
|
44
|
+
Accept a version object, ``\"1.0.0\"``, or a bare major ``1`` (→ ``1.0.0``).
|
|
45
|
+
"""
|
|
46
|
+
if isinstance(value, SemanticVersion):
|
|
47
|
+
return value
|
|
48
|
+
if isinstance(value, int):
|
|
49
|
+
return cls(major=value)
|
|
50
|
+
match = _VERSION_RE.fullmatch(safe_string(value))
|
|
51
|
+
if match is None:
|
|
52
|
+
raise ValueError(f"Invalid semantic version: {value!r}")
|
|
53
|
+
return cls(
|
|
54
|
+
major=safe_int(match.group("major")),
|
|
55
|
+
minor=safe_int(match.group("minor")),
|
|
56
|
+
patch=safe_int(match.group("patch")),
|
|
57
|
+
extra=match.group("extra"),
|
|
58
|
+
)
|
|
@@ -10,10 +10,12 @@ Set a key with ``COREKIT_SERIALIZATION__KEY`` to use an executing engine.
|
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
from corekit.serialization.enum import SerializerEngine
|
|
13
|
+
from corekit.serialization.pickle_file import PickleFileManager
|
|
13
14
|
from corekit.serialization.serializable import Serializable
|
|
14
15
|
from corekit.serialization.serializer import Serializer, SignatureError, UnsafeEngineError
|
|
15
16
|
|
|
16
17
|
__all__ = [
|
|
18
|
+
"PickleFileManager",
|
|
17
19
|
"Serializable",
|
|
18
20
|
"Serializer",
|
|
19
21
|
"SerializerEngine",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Reading and writing signed pickle files.
|
|
3
|
+
|
|
4
|
+
Pickle executes code on load. ``PickleFileManager`` therefore requires a key
|
|
5
|
+
and stores authenticated payloads via ``Serializer``, never bare ``pickle.dumps``.
|
|
6
|
+
Partial reads and line streaming are refused: a pickle document is not
|
|
7
|
+
line-oriented and a fragment is not a valid payload.
|
|
8
|
+
|
|
9
|
+
Lives under ``serialization`` rather than ``files`` so ``corekit.config`` can
|
|
10
|
+
import ``TomlFileManager`` without forming a files ↔ serialization cycle.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from typing import Any, Iterator
|
|
14
|
+
|
|
15
|
+
from corekit.files.base import FileContent, FileError, FileManager
|
|
16
|
+
from corekit.files.enum import FileMode
|
|
17
|
+
from corekit.serialization.enum import SerializerEngine
|
|
18
|
+
from corekit.serialization.serializer import Serializer
|
|
19
|
+
|
|
20
|
+
__all__ = ["PickleFileManager"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PickleFileManager(FileManager):
|
|
24
|
+
"""
|
|
25
|
+
A FileManager that round-trips values through a signed pickle ``Serializer``.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
file_path: str,
|
|
31
|
+
mode: FileMode | None = None,
|
|
32
|
+
*,
|
|
33
|
+
key: bytes | str,
|
|
34
|
+
binary: bool = True,
|
|
35
|
+
) -> None:
|
|
36
|
+
if not binary:
|
|
37
|
+
raise FileError("PickleFileManager requires binary mode; pickle payloads are bytes.")
|
|
38
|
+
super().__init__(file_path, mode=mode or FileMode.get_default(binary=True), binary=True)
|
|
39
|
+
self._serializer = Serializer(SerializerEngine.PICKLE, key=key)
|
|
40
|
+
|
|
41
|
+
def _serialize(self, content: Any) -> FileContent:
|
|
42
|
+
return self._serializer.serialize(content)
|
|
43
|
+
|
|
44
|
+
def _deserialize(self, content: FileContent) -> Any:
|
|
45
|
+
if isinstance(content, str):
|
|
46
|
+
content = content.encode("utf-8")
|
|
47
|
+
return self._serializer.deserialize(content)
|
|
48
|
+
|
|
49
|
+
def read(self, size: int = -1) -> Any:
|
|
50
|
+
"""
|
|
51
|
+
Read and verify the whole file. Partial reads are refused.
|
|
52
|
+
"""
|
|
53
|
+
if size != -1:
|
|
54
|
+
raise FileError("PickleFileManager refuses partial reads; read the whole file with size=-1.")
|
|
55
|
+
return super().read(size)
|
|
56
|
+
|
|
57
|
+
def readline(self) -> Any:
|
|
58
|
+
raise FileError("PickleFileManager cannot readline; pickle payloads are not line-oriented.")
|
|
59
|
+
|
|
60
|
+
def stream(self) -> Iterator[Any]:
|
|
61
|
+
raise FileError("PickleFileManager cannot stream; pickle payloads are not line-oriented.")
|
|
@@ -15,11 +15,28 @@ class Serializable:
|
|
|
15
15
|
|
|
16
16
|
Serializing an arbitrary object needs pickle or dill, both of which execute
|
|
17
17
|
code on load, so a key is required -- see ``Serializer``.
|
|
18
|
+
|
|
19
|
+
The HMAC key on ``_serializer`` is never written into the payload. Signing
|
|
20
|
+
authenticates provenance; it is not encryption.
|
|
18
21
|
"""
|
|
19
22
|
|
|
20
23
|
def __init__(self, **kwargs: Any) -> None:
|
|
21
24
|
self._serializer = Serializer(**kwargs)
|
|
22
25
|
|
|
26
|
+
def __getstate__(self) -> dict[str, Any]:
|
|
27
|
+
"""
|
|
28
|
+
Drop ``_serializer`` so a pickle/dill payload cannot carry the HMAC key.
|
|
29
|
+
"""
|
|
30
|
+
state = self.__dict__.copy()
|
|
31
|
+
state.pop("_serializer", None)
|
|
32
|
+
return state
|
|
33
|
+
|
|
34
|
+
def __setstate__(self, state: dict[str, Any]) -> None:
|
|
35
|
+
"""
|
|
36
|
+
Restore instance state. The serializer is reattached by ``from_serialized``.
|
|
37
|
+
"""
|
|
38
|
+
self.__dict__.update(state)
|
|
39
|
+
|
|
23
40
|
@classmethod
|
|
24
41
|
def from_serialized(cls, serialized: bytes, **kwargs: Any) -> Any:
|
|
25
42
|
"""
|
|
@@ -28,11 +45,14 @@ class Serializable:
|
|
|
28
45
|
The engine and key are supplied by the caller rather than read from the
|
|
29
46
|
payload, so the receiver decides how the bytes are decoded.
|
|
30
47
|
"""
|
|
31
|
-
|
|
48
|
+
obj = Serializer(**kwargs).deserialize(serialized)
|
|
49
|
+
if isinstance(obj, Serializable):
|
|
50
|
+
obj._serializer = Serializer(**kwargs)
|
|
51
|
+
return obj
|
|
32
52
|
|
|
33
53
|
def serialize(self) -> bytes:
|
|
34
54
|
"""
|
|
35
|
-
Encode this object.
|
|
55
|
+
Encode this object without embedding the HMAC key.
|
|
36
56
|
"""
|
|
37
57
|
if not self._serializer.is_valid_for_class():
|
|
38
58
|
raise TypeError(
|
|
@@ -24,6 +24,7 @@ from typing import Any
|
|
|
24
24
|
import dill
|
|
25
25
|
|
|
26
26
|
from corekit.config import get_settings
|
|
27
|
+
from corekit.exceptions import InternalCoreException, Retryability
|
|
27
28
|
from corekit.serialization.enum import SerializerEngine
|
|
28
29
|
|
|
29
30
|
__all__ = ["Serializer", "SignatureError", "UnsafeEngineError"]
|
|
@@ -42,17 +43,23 @@ _ENGINE_MODULES = {
|
|
|
42
43
|
}
|
|
43
44
|
|
|
44
45
|
|
|
45
|
-
class SignatureError(
|
|
46
|
+
class SignatureError(InternalCoreException):
|
|
46
47
|
"""
|
|
47
48
|
Raised when a payload's signature is missing or does not verify.
|
|
48
49
|
"""
|
|
49
50
|
|
|
51
|
+
def __init__(self, message: str, *, error: str | None = None) -> None:
|
|
52
|
+
super().__init__(message, retryable=Retryability.NON_RETRYABLE, error=error)
|
|
50
53
|
|
|
51
|
-
|
|
54
|
+
|
|
55
|
+
class UnsafeEngineError(InternalCoreException):
|
|
52
56
|
"""
|
|
53
57
|
Raised when a code-executing engine is requested without a key.
|
|
54
58
|
"""
|
|
55
59
|
|
|
60
|
+
def __init__(self, message: str, *, error: str | None = None) -> None:
|
|
61
|
+
super().__init__(message, retryable=Retryability.NON_RETRYABLE, error=error)
|
|
62
|
+
|
|
56
63
|
|
|
57
64
|
class Serializer:
|
|
58
65
|
"""
|
corekit/utils/__init__.py
CHANGED
|
@@ -9,7 +9,7 @@ configuration need.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
from corekit.utils.coercion import safe_dict, safe_float, safe_int, safe_list, safe_string, safe_tuple
|
|
12
|
-
from corekit.utils.collections import UNSET, MultiMatch, keygetter, repeated_get, split_list
|
|
12
|
+
from corekit.utils.collections import UNSET, MultiMatch, attr_or_key, keygetter, repeated_get, split_list
|
|
13
13
|
from corekit.utils.ids import (
|
|
14
14
|
SHORTCODE_CHARS,
|
|
15
15
|
generate_session_token,
|
|
@@ -29,6 +29,7 @@ __all__ = [
|
|
|
29
29
|
"UNSET",
|
|
30
30
|
"MultiMatch",
|
|
31
31
|
"Payload",
|
|
32
|
+
"attr_or_key",
|
|
32
33
|
"decode_payload",
|
|
33
34
|
"encode_payload",
|
|
34
35
|
"false_validator",
|
corekit/utils/collections.py
CHANGED
|
@@ -10,7 +10,7 @@ missing or the wrong type.
|
|
|
10
10
|
from copy import deepcopy
|
|
11
11
|
from typing import Any, Callable
|
|
12
12
|
|
|
13
|
-
__all__ = ["MultiMatch", "keygetter", "repeated_get", "split_list"]
|
|
13
|
+
__all__ = ["MultiMatch", "attr_or_key", "keygetter", "repeated_get", "split_list"]
|
|
14
14
|
|
|
15
15
|
KEY_SEPARATOR = "."
|
|
16
16
|
|
|
@@ -25,6 +25,21 @@ def split_list(items: list[Any], index: int) -> tuple[list[Any], list[Any]]:
|
|
|
25
25
|
return items[:index], items[index:]
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def attr_or_key(obj: Any, name: str) -> Any:
|
|
29
|
+
"""
|
|
30
|
+
Read ``name`` from a mapping or an object attribute.
|
|
31
|
+
|
|
32
|
+
Returns ``None`` when ``obj`` is ``None``, the key is missing, or the
|
|
33
|
+
attribute is absent. Useful for SDK payloads that arrive as either dicts
|
|
34
|
+
or attribute-bearing chunk objects.
|
|
35
|
+
"""
|
|
36
|
+
if obj is None:
|
|
37
|
+
return None
|
|
38
|
+
if isinstance(obj, dict):
|
|
39
|
+
return obj.get(name)
|
|
40
|
+
return getattr(obj, name, None)
|
|
41
|
+
|
|
42
|
+
|
|
28
43
|
class MultiMatch(dict[str, Any]):
|
|
29
44
|
"""
|
|
30
45
|
The result of a path step that matched more than one key.
|
|
@@ -56,18 +71,24 @@ class keygetter:
|
|
|
56
71
|
def call(self, data: dict[str, Any]) -> Any | MultiMatch:
|
|
57
72
|
"""
|
|
58
73
|
Apply the predicate, returning the match, a ``MultiMatch``, or ``UNSET``.
|
|
74
|
+
|
|
75
|
+
The keymap is replaced on every call. The same instance can be a step
|
|
76
|
+
more than once in one path; keeping the previous matches would make
|
|
77
|
+
the second step see keys from the first.
|
|
59
78
|
"""
|
|
79
|
+
matches = MultiMatch()
|
|
60
80
|
for key, value in data.items():
|
|
61
81
|
if self.func(key):
|
|
62
|
-
|
|
82
|
+
matches[key] = value
|
|
83
|
+
self._keymap = matches
|
|
63
84
|
|
|
64
|
-
if not
|
|
85
|
+
if not matches:
|
|
65
86
|
return UNSET
|
|
66
87
|
|
|
67
|
-
if len(
|
|
68
|
-
return next(iter(
|
|
88
|
+
if len(matches) == 1:
|
|
89
|
+
return next(iter(matches.values()))
|
|
69
90
|
|
|
70
|
-
return
|
|
91
|
+
return matches
|
|
71
92
|
|
|
72
93
|
|
|
73
94
|
def repeated_get(
|
|
@@ -91,10 +112,14 @@ def repeated_get(
|
|
|
91
112
|
|
|
92
113
|
def _get_item(current: Any, key: str | keygetter) -> Any | MultiMatch:
|
|
93
114
|
if current is UNSET or not isinstance(current, dict):
|
|
94
|
-
return
|
|
115
|
+
return UNSET
|
|
95
116
|
|
|
96
117
|
if isinstance(key, keygetter):
|
|
97
|
-
return key.
|
|
118
|
+
return key.call(current)
|
|
119
|
+
if callable(key):
|
|
120
|
+
# A bare predicate is the same step as a keygetter. A reused
|
|
121
|
+
# instance still goes through call(), which owns the keymap.
|
|
122
|
+
return keygetter(key).call(current)
|
|
98
123
|
|
|
99
124
|
return current.get(key, UNSET)
|
|
100
125
|
|
|
@@ -103,12 +128,11 @@ def repeated_get(
|
|
|
103
128
|
for key in keys:
|
|
104
129
|
if isinstance(result, MultiMatch):
|
|
105
130
|
collected = MultiMatch()
|
|
106
|
-
for
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
result = deepcopy(collected)
|
|
131
|
+
for parent_key, branch in result.items():
|
|
132
|
+
child = _get_item(branch, key)
|
|
133
|
+
if child is not UNSET:
|
|
134
|
+
collected[parent_key] = child
|
|
135
|
+
result = collected if collected else UNSET
|
|
112
136
|
else:
|
|
113
137
|
result = _get_item(result, key)
|
|
114
138
|
|
corekit/utils/payload.py
CHANGED
|
@@ -13,10 +13,14 @@ so a value that cannot cross fails at the call site where the offending
|
|
|
13
13
|
argument is still visible.
|
|
14
14
|
"""
|
|
15
15
|
|
|
16
|
+
import base64
|
|
16
17
|
import json
|
|
17
18
|
from datetime import date, datetime
|
|
19
|
+
from decimal import Decimal
|
|
18
20
|
from enum import Enum
|
|
21
|
+
from pathlib import Path
|
|
19
22
|
from typing import Any, NamedTuple
|
|
23
|
+
from uuid import UUID
|
|
20
24
|
|
|
21
25
|
__all__ = ["Payload", "decode_payload", "encode_payload"]
|
|
22
26
|
|
|
@@ -65,6 +69,14 @@ def _encode_extra(value: Any) -> Any:
|
|
|
65
69
|
return value.value
|
|
66
70
|
if isinstance(value, (datetime, date)):
|
|
67
71
|
return value.isoformat()
|
|
72
|
+
if isinstance(value, UUID):
|
|
73
|
+
return str(value)
|
|
74
|
+
if isinstance(value, Path):
|
|
75
|
+
return str(value)
|
|
76
|
+
if isinstance(value, Decimal):
|
|
77
|
+
return str(value)
|
|
78
|
+
if isinstance(value, bytes):
|
|
79
|
+
return base64.b64encode(value).decode("ascii")
|
|
68
80
|
raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
|
|
69
81
|
|
|
70
82
|
|