python-corekit 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +15 -15
- corekit/connections/sql/operations/base.py +9 -9
- corekit/connections/sql/operations/statements.py +12 -11
- corekit/connections/sql/table.py +6 -3
- corekit/data/__init__.py +6 -0
- corekit/data/arithmetic/__init__.py +97 -0
- corekit/data/arithmetic/expression.py +449 -0
- corekit/data/arithmetic/keys.py +38 -0
- corekit/data/arithmetic/operator.py +44 -0
- corekit/data/expressions/__init__.py +46 -6
- corekit/data/expressions/comparison.py +78 -1
- corekit/data/expressions/expression.py +77 -1
- corekit/data/expressions/keys.py +34 -0
- corekit/data/expressions/operator.py +33 -12
- corekit/etl/__init__.py +4 -16
- corekit/integrations/__init__.py +30 -0
- corekit/integrations/adapter.py +72 -0
- corekit/integrations/loader.py +41 -0
- corekit/integrations/normalizer.py +28 -0
- corekit/integrations/pipeline.py +136 -0
- corekit/integrations/schemas.py +49 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.4.dist-info}/METADATA +1 -1
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.4.dist-info}/RECORD +27 -16
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.4.dist-info}/WHEEL +0 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.4.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.4.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from typing import Any, Iterable
|
|
2
2
|
|
|
3
|
-
from corekit.data.expressions.expression import Expression
|
|
3
|
+
from corekit.data.expressions.expression import _FROM_DICT_LOADERS, Expression
|
|
4
|
+
from corekit.data.expressions.keys import ExpressionKey, ExpressionType
|
|
4
5
|
from corekit.data.expressions.operator import Operator
|
|
5
6
|
from corekit.data.expressions.target import Target
|
|
6
7
|
|
|
@@ -77,6 +78,50 @@ class Comparison(Expression):
|
|
|
77
78
|
"""
|
|
78
79
|
raise NotImplementedError(f"SQL cannot express `{self.symbol}`: {reason}")
|
|
79
80
|
|
|
81
|
+
def to_dict(self) -> dict[str, Any]:
|
|
82
|
+
return {
|
|
83
|
+
ExpressionKey.TYPE: self.operator.serialized,
|
|
84
|
+
ExpressionKey.LEFT: self._operand_to_dict(self.left),
|
|
85
|
+
ExpressionKey.RIGHT: self._operand_to_dict(self.right),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def from_dict(cls, data: dict[str, Any]) -> "Comparison":
|
|
90
|
+
operator = Operator.from_serialized(data[ExpressionKey.TYPE])
|
|
91
|
+
for subclass in cls.__subclasses__():
|
|
92
|
+
if subclass.operator is operator:
|
|
93
|
+
left = cls._operand_from_dict(data[ExpressionKey.LEFT])
|
|
94
|
+
right = cls._operand_from_dict(data[ExpressionKey.RIGHT])
|
|
95
|
+
if subclass is IsIn and isinstance(right, list):
|
|
96
|
+
right = frozenset(right)
|
|
97
|
+
return subclass(left, right)
|
|
98
|
+
raise ValueError(f"No comparison class registered for operator {operator!r}")
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def _operand_to_dict(value: Any) -> dict[str, Any]:
|
|
102
|
+
if isinstance(value, Expression):
|
|
103
|
+
return value.to_dict()
|
|
104
|
+
if isinstance(value, (set, frozenset)):
|
|
105
|
+
return {
|
|
106
|
+
ExpressionKey.TYPE: ExpressionType.LITERAL,
|
|
107
|
+
ExpressionKey.VALUE: sorted(value, key=repr),
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
ExpressionKey.TYPE: ExpressionType.LITERAL,
|
|
111
|
+
ExpressionKey.VALUE: value,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def _operand_from_dict(data: dict[str, Any]) -> Any:
|
|
116
|
+
if not isinstance(data, dict):
|
|
117
|
+
raise TypeError(f"Comparison operand must be a dict, got {type(data)!r}")
|
|
118
|
+
kind = data.get(ExpressionKey.TYPE)
|
|
119
|
+
if kind == ExpressionType.FIELD:
|
|
120
|
+
return FieldExpression.from_dict(data)
|
|
121
|
+
if kind == ExpressionType.LITERAL:
|
|
122
|
+
return data.get(ExpressionKey.VALUE)
|
|
123
|
+
raise ValueError(f"Unknown comparison operand type {kind!r}")
|
|
124
|
+
|
|
80
125
|
|
|
81
126
|
class Equals(Comparison):
|
|
82
127
|
operator = Operator.EQUALS
|
|
@@ -260,6 +305,19 @@ class ValueExpression(Expression):
|
|
|
260
305
|
"""
|
|
261
306
|
return self.value
|
|
262
307
|
|
|
308
|
+
def to_dict(self) -> dict[str, Any]:
|
|
309
|
+
value = self.value
|
|
310
|
+
if isinstance(value, (set, frozenset)):
|
|
311
|
+
value = sorted(value, key=repr)
|
|
312
|
+
return {
|
|
313
|
+
ExpressionKey.TYPE: ExpressionType.LITERAL,
|
|
314
|
+
ExpressionKey.VALUE: value,
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
@classmethod
|
|
318
|
+
def from_dict(cls, data: dict[str, Any]) -> "ValueExpression":
|
|
319
|
+
return cls(data.get(ExpressionKey.VALUE))
|
|
320
|
+
|
|
263
321
|
|
|
264
322
|
class FieldExpression(ValueExpression):
|
|
265
323
|
"""
|
|
@@ -300,3 +358,22 @@ class FieldExpression(ValueExpression):
|
|
|
300
358
|
if value is UNSET:
|
|
301
359
|
raise UnknownFieldError(f"{context!r} has no field {self.value!r}")
|
|
302
360
|
return value
|
|
361
|
+
|
|
362
|
+
def to_dict(self) -> dict[str, Any]:
|
|
363
|
+
return {
|
|
364
|
+
ExpressionKey.TYPE: ExpressionType.FIELD,
|
|
365
|
+
ExpressionKey.NAME: self.value,
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
@classmethod
|
|
369
|
+
def from_dict(cls, data: dict[str, Any]) -> "FieldExpression":
|
|
370
|
+
name = data.get(ExpressionKey.NAME)
|
|
371
|
+
if not isinstance(name, str) or not name:
|
|
372
|
+
raise ValueError("Field expression requires a non-empty string name")
|
|
373
|
+
return cls(name)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
_FROM_DICT_LOADERS[ExpressionType.FIELD] = FieldExpression.from_dict
|
|
377
|
+
_FROM_DICT_LOADERS[ExpressionType.LITERAL] = ValueExpression.from_dict
|
|
378
|
+
for _operator in Operator:
|
|
379
|
+
_FROM_DICT_LOADERS[_operator.serialized] = Comparison.from_dict
|
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
1
4
|
from typing import Any
|
|
2
5
|
|
|
6
|
+
from corekit.data.expressions.keys import ExpressionKey, ExpressionType
|
|
7
|
+
|
|
8
|
+
# Filled by comparison.py at import time so from_dict needs no nested imports.
|
|
9
|
+
_FROM_DICT_LOADERS: dict[str, Callable[[dict[str, Any]], Expression]] = {}
|
|
10
|
+
|
|
3
11
|
|
|
4
12
|
class Expression:
|
|
5
13
|
"""
|
|
6
|
-
Base class for composable predicates. Subclasses implement __call__
|
|
14
|
+
Base class for composable predicates. Subclasses implement __call__,
|
|
15
|
+
``to_sqlalchemy``, and ``to_dict``.
|
|
7
16
|
"""
|
|
8
17
|
|
|
9
18
|
def __call__(self, *args: Any, **kwargs: Any) -> bool:
|
|
@@ -26,6 +35,35 @@ class Expression:
|
|
|
26
35
|
"""
|
|
27
36
|
raise NotImplementedError
|
|
28
37
|
|
|
38
|
+
def to_dict(self) -> dict[str, Any]:
|
|
39
|
+
"""
|
|
40
|
+
JSON-safe encoding of this node (and its children).
|
|
41
|
+
"""
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def from_dict(cls, data: dict[str, Any] | "Expression") -> "Expression":
|
|
46
|
+
"""
|
|
47
|
+
Rebuild an expression tree from ``to_dict`` output.
|
|
48
|
+
"""
|
|
49
|
+
if isinstance(data, Expression):
|
|
50
|
+
return data
|
|
51
|
+
if not isinstance(data, dict):
|
|
52
|
+
raise TypeError(f"Expression.from_dict expected a dict, got {type(data)!r}")
|
|
53
|
+
|
|
54
|
+
kind = data.get(ExpressionKey.TYPE)
|
|
55
|
+
if kind == ExpressionType.AND:
|
|
56
|
+
return And.from_dict(data)
|
|
57
|
+
if kind == ExpressionType.OR:
|
|
58
|
+
return Or.from_dict(data)
|
|
59
|
+
if kind == ExpressionType.NOT:
|
|
60
|
+
return Not.from_dict(data)
|
|
61
|
+
|
|
62
|
+
loader = _FROM_DICT_LOADERS.get(kind) if isinstance(kind, str) else None
|
|
63
|
+
if loader is not None:
|
|
64
|
+
return loader(data)
|
|
65
|
+
raise ValueError(f"Unknown expression type {kind!r}")
|
|
66
|
+
|
|
29
67
|
|
|
30
68
|
class And(Expression):
|
|
31
69
|
def __init__(self, left: Expression, right: Expression) -> None:
|
|
@@ -41,6 +79,20 @@ class And(Expression):
|
|
|
41
79
|
def to_sqlalchemy(self, context: Any) -> Any:
|
|
42
80
|
return self.left.to_sqlalchemy(context) & self.right.to_sqlalchemy(context)
|
|
43
81
|
|
|
82
|
+
def to_dict(self) -> dict[str, Any]:
|
|
83
|
+
return {
|
|
84
|
+
ExpressionKey.TYPE: ExpressionType.AND,
|
|
85
|
+
ExpressionKey.LEFT: self.left.to_dict(),
|
|
86
|
+
ExpressionKey.RIGHT: self.right.to_dict(),
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def from_dict(cls, data: dict[str, Any]) -> "And":
|
|
91
|
+
return cls(
|
|
92
|
+
Expression.from_dict(data[ExpressionKey.LEFT]),
|
|
93
|
+
Expression.from_dict(data[ExpressionKey.RIGHT]),
|
|
94
|
+
)
|
|
95
|
+
|
|
44
96
|
|
|
45
97
|
class Or(Expression):
|
|
46
98
|
def __init__(self, left: Expression, right: Expression) -> None:
|
|
@@ -56,6 +108,20 @@ class Or(Expression):
|
|
|
56
108
|
def to_sqlalchemy(self, context: Any) -> Any:
|
|
57
109
|
return self.left.to_sqlalchemy(context) | self.right.to_sqlalchemy(context)
|
|
58
110
|
|
|
111
|
+
def to_dict(self) -> dict[str, Any]:
|
|
112
|
+
return {
|
|
113
|
+
ExpressionKey.TYPE: ExpressionType.OR,
|
|
114
|
+
ExpressionKey.LEFT: self.left.to_dict(),
|
|
115
|
+
ExpressionKey.RIGHT: self.right.to_dict(),
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def from_dict(cls, data: dict[str, Any]) -> "Or":
|
|
120
|
+
return cls(
|
|
121
|
+
Expression.from_dict(data[ExpressionKey.LEFT]),
|
|
122
|
+
Expression.from_dict(data[ExpressionKey.RIGHT]),
|
|
123
|
+
)
|
|
124
|
+
|
|
59
125
|
|
|
60
126
|
class Not(Expression):
|
|
61
127
|
def __init__(self, expr: Expression) -> None:
|
|
@@ -69,3 +135,13 @@ class Not(Expression):
|
|
|
69
135
|
|
|
70
136
|
def to_sqlalchemy(self, context: Any) -> Any:
|
|
71
137
|
return ~self.expr.to_sqlalchemy(context)
|
|
138
|
+
|
|
139
|
+
def to_dict(self) -> dict[str, Any]:
|
|
140
|
+
return {
|
|
141
|
+
ExpressionKey.TYPE: ExpressionType.NOT,
|
|
142
|
+
ExpressionKey.EXPR: self.expr.to_dict(),
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def from_dict(cls, data: dict[str, Any]) -> "Not":
|
|
147
|
+
return cls(Expression.from_dict(data[ExpressionKey.EXPR]))
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Stable serialization tags and dict keys for expression trees.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from corekit.schemas.enum import StringEnum
|
|
6
|
+
|
|
7
|
+
__all__ = ["ExpressionKey", "ExpressionType"]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ExpressionType(StringEnum):
|
|
11
|
+
"""
|
|
12
|
+
Node kind for non-comparison expression nodes.
|
|
13
|
+
|
|
14
|
+
Comparison nodes use ``Operator.serialized`` (from ``Dialect.name``) instead.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
AND = "and"
|
|
18
|
+
OR = "or"
|
|
19
|
+
NOT = "not"
|
|
20
|
+
FIELD = "field"
|
|
21
|
+
LITERAL = "literal"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ExpressionKey(StringEnum):
|
|
25
|
+
"""
|
|
26
|
+
Dict keys used by ``Expression.to_dict`` / ``from_dict``.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
TYPE = "type"
|
|
30
|
+
LEFT = "left"
|
|
31
|
+
RIGHT = "right"
|
|
32
|
+
EXPR = "expr"
|
|
33
|
+
NAME = "name"
|
|
34
|
+
VALUE = "value"
|
|
@@ -11,13 +11,15 @@ __all__ = ["Dialect", "Operator"]
|
|
|
11
11
|
|
|
12
12
|
class Dialect(NamedTuple):
|
|
13
13
|
"""
|
|
14
|
-
How one operator is spelled for humans.
|
|
14
|
+
How one operator is spelled for humans and for JSON.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
``symbol`` is the infix form (``>=``). ``name`` is the stable serialization
|
|
17
|
+
key (``greater_than_or_equals``). SQL is compiled through SQLAlchemy, not
|
|
18
|
+
spelled here.
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
21
|
symbol: str
|
|
22
|
+
name: str
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
class Operator(ValidatingEnum):
|
|
@@ -25,16 +27,35 @@ class Operator(ValidatingEnum):
|
|
|
25
27
|
A comparison operator, identified by the comparison class that uses it.
|
|
26
28
|
"""
|
|
27
29
|
|
|
28
|
-
EQUALS = Dialect("==")
|
|
29
|
-
NOT_EQUALS = Dialect("!=")
|
|
30
|
-
LESS_THAN = Dialect("<")
|
|
31
|
-
LESS_THAN_OR_EQUALS = Dialect("<=")
|
|
32
|
-
GREATER_THAN = Dialect(">")
|
|
33
|
-
GREATER_THAN_OR_EQUALS = Dialect(">=")
|
|
34
|
-
IS_IN = Dialect("is in")
|
|
35
|
-
CONTAINS = Dialect("contains")
|
|
36
|
-
ICONTAINS = Dialect("icontains")
|
|
30
|
+
EQUALS = Dialect("==", "equals")
|
|
31
|
+
NOT_EQUALS = Dialect("!=", "not_equals")
|
|
32
|
+
LESS_THAN = Dialect("<", "less_than")
|
|
33
|
+
LESS_THAN_OR_EQUALS = Dialect("<=", "less_than_or_equals")
|
|
34
|
+
GREATER_THAN = Dialect(">", "greater_than")
|
|
35
|
+
GREATER_THAN_OR_EQUALS = Dialect(">=", "greater_than_or_equals")
|
|
36
|
+
IS_IN = Dialect("is in", "is_in")
|
|
37
|
+
CONTAINS = Dialect("contains", "contains")
|
|
38
|
+
ICONTAINS = Dialect("icontains", "icontains")
|
|
37
39
|
|
|
38
40
|
@property
|
|
39
41
|
def symbol(self) -> str:
|
|
40
42
|
return self.value.symbol
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def serialized(self) -> str:
|
|
46
|
+
"""
|
|
47
|
+
Stable type tag for ``to_dict`` / ``from_dict`` (``Dialect.name``).
|
|
48
|
+
|
|
49
|
+
Named ``serialized`` so it does not shadow ``Enum.name``.
|
|
50
|
+
"""
|
|
51
|
+
return self.value.name
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def from_serialized(cls, name: str) -> "Operator":
|
|
55
|
+
"""
|
|
56
|
+
Resolve an operator from its ``Dialect.name`` / ``serialized`` tag.
|
|
57
|
+
"""
|
|
58
|
+
for member in cls:
|
|
59
|
+
if member.serialized == name:
|
|
60
|
+
return member
|
|
61
|
+
raise ValueError(f"Unknown operator serialization {name!r}")
|
corekit/etl/__init__.py
CHANGED
|
@@ -1,21 +1,9 @@
|
|
|
1
1
|
"""
|
|
2
|
-
ETL pipelines.
|
|
2
|
+
ETL pipelines (legacy).
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class UserPipeline(BaseETLOrchestrator):
|
|
8
|
-
'''
|
|
9
|
-
Loads users from the API into the warehouse.
|
|
10
|
-
'''
|
|
11
|
-
extractor = UserExtractor
|
|
12
|
-
transformer = UserTransformer
|
|
13
|
-
loader = UserLoader
|
|
14
|
-
|
|
15
|
-
await UserPipeline().run()
|
|
16
|
-
|
|
17
|
-
The run loop streams, so memory use is bounded by the batch size rather than by
|
|
18
|
-
how much data the source holds.
|
|
4
|
+
Prefer ``corekit.integrations`` for new work: ``BaseAdapter`` / ``BaseNormalizer``
|
|
5
|
+
/ ``BaseLoader`` / ``BasePipeline``. This package remains for existing tests and
|
|
6
|
+
imports; new vendors should not extend Extractor/Transformer naming.
|
|
19
7
|
"""
|
|
20
8
|
|
|
21
9
|
from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Integration framework: clients, adapters, normalizers, loaders, pipelines.
|
|
3
|
+
|
|
4
|
+
Layering::
|
|
5
|
+
|
|
6
|
+
BaseApiClient — wire + auth to the external system
|
|
7
|
+
BaseNormalizer — raw payload → concrete BaseModel (never dict-out)
|
|
8
|
+
BaseAdapter — named ops (interactive) + optional bulk sources (sync)
|
|
9
|
+
BaseLoader — persist normalized items into the host system
|
|
10
|
+
BasePipeline — stream adapter bulk → loader (nightly / full sync)
|
|
11
|
+
|
|
12
|
+
Interactive Integrations call ``adapter.run(...)``. Catalog syncs call
|
|
13
|
+
``pipeline.run()``. Do not route point lookups through the pipeline.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from corekit.integrations.adapter import BaseAdapter
|
|
17
|
+
from corekit.integrations.loader import BaseLoader
|
|
18
|
+
from corekit.integrations.normalizer import BaseNormalizer
|
|
19
|
+
from corekit.integrations.pipeline import BasePipeline
|
|
20
|
+
from corekit.integrations.schemas import LoadableBatch, NormalizedItem, OperationClass
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BaseAdapter",
|
|
24
|
+
"BaseLoader",
|
|
25
|
+
"BaseNormalizer",
|
|
26
|
+
"BasePipeline",
|
|
27
|
+
"LoadableBatch",
|
|
28
|
+
"NormalizedItem",
|
|
29
|
+
"OperationClass",
|
|
30
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Vendor adapter: named operations plus optional bulk sources for sync pipelines.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from collections.abc import AsyncIterable, Callable
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
from corekit.http import BadRequestException, BaseApiClient
|
|
14
|
+
from corekit.integrations.normalizer import BaseNormalizer
|
|
15
|
+
from corekit.integrations.schemas import NormalizedItem, OperationClass
|
|
16
|
+
from corekit.observability import Loggable
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BaseAdapter(Loggable, ABC):
|
|
20
|
+
"""
|
|
21
|
+
Integration surface for one external system.
|
|
22
|
+
|
|
23
|
+
- **Named ops** (``run``): interactive request/response (search, create, …).
|
|
24
|
+
- **Bulk sources** (``iter_bulk``): streaming catalog/sync pulls for pipelines.
|
|
25
|
+
|
|
26
|
+
Always constructed with a ``BaseApiClient`` and a ``BaseNormalizer``. Ops and
|
|
27
|
+
bulk sources return ``BaseModel`` instances — never raw dicts.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, client: BaseApiClient, normalizer: BaseNormalizer[Any]) -> None:
|
|
31
|
+
super().__init__()
|
|
32
|
+
self.client = client
|
|
33
|
+
self.normalizer = normalizer
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def operation_classes(self) -> dict[str, OperationClass]:
|
|
37
|
+
"""
|
|
38
|
+
Map operation key → read/write class.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def operations(self) -> frozenset[str]:
|
|
43
|
+
return frozenset(self.operation_classes())
|
|
44
|
+
|
|
45
|
+
def operation_class(self, operation: str) -> OperationClass | None:
|
|
46
|
+
return self.operation_classes().get(operation)
|
|
47
|
+
|
|
48
|
+
def require_operation(self, operation: str) -> OperationClass:
|
|
49
|
+
op_class = self.operation_class(operation)
|
|
50
|
+
if op_class is None:
|
|
51
|
+
raise BadRequestException(detail=f"Unknown operation '{operation}'")
|
|
52
|
+
return op_class
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
async def run(self, operation: str, payload: dict[str, Any], **context: Any) -> BaseModel:
|
|
56
|
+
"""
|
|
57
|
+
Execute a named operation. Return a normalized ``BaseModel`` (never secrets).
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def bulk_sources(self) -> list[Callable[..., AsyncIterable[NormalizedItem]]]:
|
|
61
|
+
"""
|
|
62
|
+
Optional streaming sources for full sync. Default: none.
|
|
63
|
+
"""
|
|
64
|
+
return []
|
|
65
|
+
|
|
66
|
+
async def iter_bulk(self, *args: Any, **kwargs: Any) -> AsyncIterable[NormalizedItem]:
|
|
67
|
+
"""
|
|
68
|
+
Yield normalized items from every bulk source.
|
|
69
|
+
"""
|
|
70
|
+
for source in self.bulk_sources():
|
|
71
|
+
async for item in source(*args, **kwargs):
|
|
72
|
+
yield item
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Persist normalized items into the host system.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
|
|
9
|
+
from corekit.integrations.schemas import LoadableBatch, NormalizedItem
|
|
10
|
+
from corekit.observability import Loggable
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseLoader(Loggable, ABC):
|
|
14
|
+
"""
|
|
15
|
+
Batches normalized items and writes them to a destination (DB, namespace, …).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
super().__init__()
|
|
20
|
+
self.batch: LoadableBatch = LoadableBatch()
|
|
21
|
+
|
|
22
|
+
def _should_load(self, min_ops: int) -> bool:
|
|
23
|
+
return bool(len(self.batch)) and len(self.batch) >= min_ops
|
|
24
|
+
|
|
25
|
+
def add_item(self, item: NormalizedItem) -> None:
|
|
26
|
+
self.batch.add_item(item)
|
|
27
|
+
|
|
28
|
+
async def flush(self) -> None:
|
|
29
|
+
await self.load(min_ops=0)
|
|
30
|
+
|
|
31
|
+
async def load(self, min_ops: int) -> None:
|
|
32
|
+
if not self._should_load(min_ops):
|
|
33
|
+
return
|
|
34
|
+
await self._load(self.batch)
|
|
35
|
+
self.batch.clear()
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def _load(self, batch: LoadableBatch) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Write a batch to its destination.
|
|
41
|
+
"""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Translate raw external payloads into known Pydantic models.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from typing import Any, Generic, TypeVar
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
|
|
12
|
+
from corekit.observability import Loggable
|
|
13
|
+
|
|
14
|
+
TModel = TypeVar("TModel", bound=BaseModel)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BaseNormalizer(Loggable, ABC, Generic[TModel]):
|
|
18
|
+
"""
|
|
19
|
+
Pure translation from vendor/raw payloads to a concrete ``BaseModel``.
|
|
20
|
+
|
|
21
|
+
Output is always structured. Dict-in/dict-out is not a normalizer.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def normalize(self, raw: Any) -> TModel:
|
|
26
|
+
"""
|
|
27
|
+
Return a typed model downstream code can rely on.
|
|
28
|
+
"""
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bulk sync: stream adapter bulk sources into a loader.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, ClassVar
|
|
9
|
+
|
|
10
|
+
from corekit.integrations.adapter import BaseAdapter
|
|
11
|
+
from corekit.integrations.loader import BaseLoader
|
|
12
|
+
from corekit.observability import Loggable
|
|
13
|
+
from corekit.registry import SmartRegistry
|
|
14
|
+
|
|
15
|
+
__all__ = ["BasePipeline"]
|
|
16
|
+
|
|
17
|
+
DEFAULT_BATCH_SIZE = 100
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class PipelineStage:
|
|
22
|
+
"""
|
|
23
|
+
One declared stage on a pipeline class.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
attribute: str
|
|
27
|
+
base_class: type
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def label(self) -> str:
|
|
31
|
+
return self.base_class.__name__.removeprefix("Base")
|
|
32
|
+
|
|
33
|
+
def declared_on(self, cls: type) -> Any:
|
|
34
|
+
return cls.__dict__.get(self.attribute)
|
|
35
|
+
|
|
36
|
+
def validate(self, cls: type) -> None:
|
|
37
|
+
stage = getattr(cls, self.attribute, None)
|
|
38
|
+
if stage is None:
|
|
39
|
+
raise TypeError(f"{cls.__name__} is missing its {self.attribute}. Set {self.attribute} = Your{self.label}.")
|
|
40
|
+
if not (isinstance(stage, type) and issubclass(stage, self.base_class)):
|
|
41
|
+
raise TypeError(
|
|
42
|
+
f"{cls.__name__}.{self.attribute} must be a {self.base_class.__name__} subclass, "
|
|
43
|
+
f"got {stage!r}. Assign the class itself, not an instance."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class BasePipeline(Loggable):
|
|
48
|
+
"""
|
|
49
|
+
Runs adapter bulk sync into a loader.
|
|
50
|
+
|
|
51
|
+
Interactive named ops stay on ``BaseAdapter.run`` — this class is for
|
|
52
|
+
nightly/full pulls only.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
adapter: ClassVar[type[BaseAdapter] | None] = None
|
|
56
|
+
loader: ClassVar[type[BaseLoader] | None] = None
|
|
57
|
+
batch_size: ClassVar[int] = DEFAULT_BATCH_SIZE
|
|
58
|
+
|
|
59
|
+
__registry__: SmartRegistry = SmartRegistry()
|
|
60
|
+
__binding_only__: ClassVar[bool] = False
|
|
61
|
+
|
|
62
|
+
STAGES: ClassVar[list[PipelineStage]] = [
|
|
63
|
+
PipelineStage(attribute="adapter", base_class=BaseAdapter),
|
|
64
|
+
PipelineStage(attribute="loader", base_class=BaseLoader),
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
68
|
+
super().__init_subclass__(**kwargs)
|
|
69
|
+
if cls.__dict__.get("__binding_only__"):
|
|
70
|
+
return
|
|
71
|
+
if not any(stage.declared_on(cls) for stage in cls.STAGES):
|
|
72
|
+
return
|
|
73
|
+
for stage in cls.STAGES:
|
|
74
|
+
stage.validate(cls)
|
|
75
|
+
BasePipeline.__registry__[cls.__name__] = cls
|
|
76
|
+
|
|
77
|
+
def __class_getitem__(cls, item: tuple[type, ...]) -> Any:
|
|
78
|
+
if not isinstance(item, tuple) or len(item) != 2:
|
|
79
|
+
raise TypeError(f"{cls.__name__}[...] takes exactly two stages: {cls.__name__}[Adapter, Loader]")
|
|
80
|
+
bound = {stage.attribute: value for stage, value in zip(cls.STAGES, item, strict=True)}
|
|
81
|
+
return type(f"{cls.__name__}_{item[0].__name__}", (cls,), bound)
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def get_pipeline_types(cls) -> SmartRegistry:
|
|
85
|
+
return cls.__registry__
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def get_pipeline_by_name(cls, name: str) -> type[BasePipeline] | None:
|
|
89
|
+
return cls.__registry__.get(name)
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
adapter: BaseAdapter | None = None,
|
|
94
|
+
loader: BaseLoader | None = None,
|
|
95
|
+
batch_size: int | None = None,
|
|
96
|
+
on_item_error: str = "raise",
|
|
97
|
+
) -> None:
|
|
98
|
+
if on_item_error not in ("raise", "skip"):
|
|
99
|
+
raise ValueError("on_item_error must be 'raise' or 'skip'")
|
|
100
|
+
super().__init__()
|
|
101
|
+
self.batch_size = batch_size if batch_size is not None else type(self).batch_size
|
|
102
|
+
self.on_item_error = on_item_error
|
|
103
|
+
self._adapter = adapter
|
|
104
|
+
self._loader = loader
|
|
105
|
+
|
|
106
|
+
def _build(self, override: Any, name: str) -> Any:
|
|
107
|
+
if override is not None:
|
|
108
|
+
return override
|
|
109
|
+
stage_class = getattr(type(self), name, None)
|
|
110
|
+
if stage_class is None:
|
|
111
|
+
raise TypeError(f"{type(self).__name__} has no {name}. Declare one, or pass {name}=...")
|
|
112
|
+
return stage_class()
|
|
113
|
+
|
|
114
|
+
async def run(self) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Stream ``adapter.iter_bulk()`` into the loader until exhausted.
|
|
117
|
+
"""
|
|
118
|
+
adapter = self._build(self._adapter, "adapter")
|
|
119
|
+
loader = self._build(self._loader, "loader")
|
|
120
|
+
|
|
121
|
+
count = 0
|
|
122
|
+
skipped = 0
|
|
123
|
+
async for item in adapter.iter_bulk():
|
|
124
|
+
try:
|
|
125
|
+
loader.add_item(item)
|
|
126
|
+
except Exception:
|
|
127
|
+
if self.on_item_error == "raise":
|
|
128
|
+
raise
|
|
129
|
+
skipped += 1
|
|
130
|
+
self.exception(f"{type(self).__name__} skipped an item")
|
|
131
|
+
continue
|
|
132
|
+
await loader.load(min_ops=self.batch_size)
|
|
133
|
+
count += 1
|
|
134
|
+
|
|
135
|
+
await loader.flush()
|
|
136
|
+
self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")
|