python-corekit 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  from typing import Any, Iterable
2
2
 
3
- from corekit.data.expressions.expression import Expression
3
+ from corekit.data.expressions.expression import _FROM_DICT_LOADERS, Expression
4
+ from corekit.data.expressions.keys import ExpressionKey, ExpressionType
4
5
  from corekit.data.expressions.operator import Operator
5
6
  from corekit.data.expressions.target import Target
6
7
 
@@ -77,6 +78,50 @@ class Comparison(Expression):
77
78
  """
78
79
  raise NotImplementedError(f"SQL cannot express `{self.symbol}`: {reason}")
79
80
 
81
+ def to_dict(self) -> dict[str, Any]:
82
+ return {
83
+ ExpressionKey.TYPE: self.operator.serialized,
84
+ ExpressionKey.LEFT: self._operand_to_dict(self.left),
85
+ ExpressionKey.RIGHT: self._operand_to_dict(self.right),
86
+ }
87
+
88
+ @classmethod
89
+ def from_dict(cls, data: dict[str, Any]) -> "Comparison":
90
+ operator = Operator.from_serialized(data[ExpressionKey.TYPE])
91
+ for subclass in cls.__subclasses__():
92
+ if subclass.operator is operator:
93
+ left = cls._operand_from_dict(data[ExpressionKey.LEFT])
94
+ right = cls._operand_from_dict(data[ExpressionKey.RIGHT])
95
+ if subclass is IsIn and isinstance(right, list):
96
+ right = frozenset(right)
97
+ return subclass(left, right)
98
+ raise ValueError(f"No comparison class registered for operator {operator!r}")
99
+
100
+ @staticmethod
101
+ def _operand_to_dict(value: Any) -> dict[str, Any]:
102
+ if isinstance(value, Expression):
103
+ return value.to_dict()
104
+ if isinstance(value, (set, frozenset)):
105
+ return {
106
+ ExpressionKey.TYPE: ExpressionType.LITERAL,
107
+ ExpressionKey.VALUE: sorted(value, key=repr),
108
+ }
109
+ return {
110
+ ExpressionKey.TYPE: ExpressionType.LITERAL,
111
+ ExpressionKey.VALUE: value,
112
+ }
113
+
114
+ @staticmethod
115
+ def _operand_from_dict(data: dict[str, Any]) -> Any:
116
+ if not isinstance(data, dict):
117
+ raise TypeError(f"Comparison operand must be a dict, got {type(data)!r}")
118
+ kind = data.get(ExpressionKey.TYPE)
119
+ if kind == ExpressionType.FIELD:
120
+ return FieldExpression.from_dict(data)
121
+ if kind == ExpressionType.LITERAL:
122
+ return data.get(ExpressionKey.VALUE)
123
+ raise ValueError(f"Unknown comparison operand type {kind!r}")
124
+
80
125
 
81
126
  class Equals(Comparison):
82
127
  operator = Operator.EQUALS
@@ -260,6 +305,19 @@ class ValueExpression(Expression):
260
305
  """
261
306
  return self.value
262
307
 
308
+ def to_dict(self) -> dict[str, Any]:
309
+ value = self.value
310
+ if isinstance(value, (set, frozenset)):
311
+ value = sorted(value, key=repr)
312
+ return {
313
+ ExpressionKey.TYPE: ExpressionType.LITERAL,
314
+ ExpressionKey.VALUE: value,
315
+ }
316
+
317
+ @classmethod
318
+ def from_dict(cls, data: dict[str, Any]) -> "ValueExpression":
319
+ return cls(data.get(ExpressionKey.VALUE))
320
+
263
321
 
264
322
  class FieldExpression(ValueExpression):
265
323
  """
@@ -300,3 +358,22 @@ class FieldExpression(ValueExpression):
300
358
  if value is UNSET:
301
359
  raise UnknownFieldError(f"{context!r} has no field {self.value!r}")
302
360
  return value
361
+
362
+ def to_dict(self) -> dict[str, Any]:
363
+ return {
364
+ ExpressionKey.TYPE: ExpressionType.FIELD,
365
+ ExpressionKey.NAME: self.value,
366
+ }
367
+
368
+ @classmethod
369
+ def from_dict(cls, data: dict[str, Any]) -> "FieldExpression":
370
+ name = data.get(ExpressionKey.NAME)
371
+ if not isinstance(name, str) or not name:
372
+ raise ValueError("Field expression requires a non-empty string name")
373
+ return cls(name)
374
+
375
+
376
+ _FROM_DICT_LOADERS[ExpressionType.FIELD] = FieldExpression.from_dict
377
+ _FROM_DICT_LOADERS[ExpressionType.LITERAL] = ValueExpression.from_dict
378
+ for _operator in Operator:
379
+ _FROM_DICT_LOADERS[_operator.serialized] = Comparison.from_dict
@@ -1,9 +1,18 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
1
4
  from typing import Any
2
5
 
6
+ from corekit.data.expressions.keys import ExpressionKey, ExpressionType
7
+
8
+ # Filled by comparison.py at import time so from_dict needs no nested imports.
9
+ _FROM_DICT_LOADERS: dict[str, Callable[[dict[str, Any]], Expression]] = {}
10
+
3
11
 
4
12
  class Expression:
5
13
  """
6
- Base class for composable predicates. Subclasses implement __call__
14
+ Base class for composable predicates. Subclasses implement __call__,
15
+ ``to_sqlalchemy``, and ``to_dict``.
7
16
  """
8
17
 
9
18
  def __call__(self, *args: Any, **kwargs: Any) -> bool:
@@ -26,6 +35,35 @@ class Expression:
26
35
  """
27
36
  raise NotImplementedError
28
37
 
38
+ def to_dict(self) -> dict[str, Any]:
39
+ """
40
+ JSON-safe encoding of this node (and its children).
41
+ """
42
+ raise NotImplementedError
43
+
44
+ @classmethod
45
+ def from_dict(cls, data: dict[str, Any] | "Expression") -> "Expression":
46
+ """
47
+ Rebuild an expression tree from ``to_dict`` output.
48
+ """
49
+ if isinstance(data, Expression):
50
+ return data
51
+ if not isinstance(data, dict):
52
+ raise TypeError(f"Expression.from_dict expected a dict, got {type(data)!r}")
53
+
54
+ kind = data.get(ExpressionKey.TYPE)
55
+ if kind == ExpressionType.AND:
56
+ return And.from_dict(data)
57
+ if kind == ExpressionType.OR:
58
+ return Or.from_dict(data)
59
+ if kind == ExpressionType.NOT:
60
+ return Not.from_dict(data)
61
+
62
+ loader = _FROM_DICT_LOADERS.get(kind) if isinstance(kind, str) else None
63
+ if loader is not None:
64
+ return loader(data)
65
+ raise ValueError(f"Unknown expression type {kind!r}")
66
+
29
67
 
30
68
  class And(Expression):
31
69
  def __init__(self, left: Expression, right: Expression) -> None:
@@ -41,6 +79,20 @@ class And(Expression):
41
79
  def to_sqlalchemy(self, context: Any) -> Any:
42
80
  return self.left.to_sqlalchemy(context) & self.right.to_sqlalchemy(context)
43
81
 
82
+ def to_dict(self) -> dict[str, Any]:
83
+ return {
84
+ ExpressionKey.TYPE: ExpressionType.AND,
85
+ ExpressionKey.LEFT: self.left.to_dict(),
86
+ ExpressionKey.RIGHT: self.right.to_dict(),
87
+ }
88
+
89
+ @classmethod
90
+ def from_dict(cls, data: dict[str, Any]) -> "And":
91
+ return cls(
92
+ Expression.from_dict(data[ExpressionKey.LEFT]),
93
+ Expression.from_dict(data[ExpressionKey.RIGHT]),
94
+ )
95
+
44
96
 
45
97
  class Or(Expression):
46
98
  def __init__(self, left: Expression, right: Expression) -> None:
@@ -56,6 +108,20 @@ class Or(Expression):
56
108
  def to_sqlalchemy(self, context: Any) -> Any:
57
109
  return self.left.to_sqlalchemy(context) | self.right.to_sqlalchemy(context)
58
110
 
111
+ def to_dict(self) -> dict[str, Any]:
112
+ return {
113
+ ExpressionKey.TYPE: ExpressionType.OR,
114
+ ExpressionKey.LEFT: self.left.to_dict(),
115
+ ExpressionKey.RIGHT: self.right.to_dict(),
116
+ }
117
+
118
+ @classmethod
119
+ def from_dict(cls, data: dict[str, Any]) -> "Or":
120
+ return cls(
121
+ Expression.from_dict(data[ExpressionKey.LEFT]),
122
+ Expression.from_dict(data[ExpressionKey.RIGHT]),
123
+ )
124
+
59
125
 
60
126
  class Not(Expression):
61
127
  def __init__(self, expr: Expression) -> None:
@@ -69,3 +135,13 @@ class Not(Expression):
69
135
 
70
136
  def to_sqlalchemy(self, context: Any) -> Any:
71
137
  return ~self.expr.to_sqlalchemy(context)
138
+
139
+ def to_dict(self) -> dict[str, Any]:
140
+ return {
141
+ ExpressionKey.TYPE: ExpressionType.NOT,
142
+ ExpressionKey.EXPR: self.expr.to_dict(),
143
+ }
144
+
145
+ @classmethod
146
+ def from_dict(cls, data: dict[str, Any]) -> "Not":
147
+ return cls(Expression.from_dict(data[ExpressionKey.EXPR]))
@@ -0,0 +1,34 @@
1
+ """
2
+ Stable serialization tags and dict keys for expression trees.
3
+ """
4
+
5
+ from corekit.schemas.enum import StringEnum
6
+
7
+ __all__ = ["ExpressionKey", "ExpressionType"]
8
+
9
+
10
+ class ExpressionType(StringEnum):
11
+ """
12
+ Node kind for non-comparison expression nodes.
13
+
14
+ Comparison nodes use ``Operator.serialized`` (from ``Dialect.name``) instead.
15
+ """
16
+
17
+ AND = "and"
18
+ OR = "or"
19
+ NOT = "not"
20
+ FIELD = "field"
21
+ LITERAL = "literal"
22
+
23
+
24
+ class ExpressionKey(StringEnum):
25
+ """
26
+ Dict keys used by ``Expression.to_dict`` / ``from_dict``.
27
+ """
28
+
29
+ TYPE = "type"
30
+ LEFT = "left"
31
+ RIGHT = "right"
32
+ EXPR = "expr"
33
+ NAME = "name"
34
+ VALUE = "value"
@@ -11,13 +11,15 @@ __all__ = ["Dialect", "Operator"]
11
11
 
12
12
  class Dialect(NamedTuple):
13
13
  """
14
- How one operator is spelled for humans.
14
+ How one operator is spelled for humans and for JSON.
15
15
 
16
- SQL is compiled through SQLAlchemy, not spelled here. A backend that does
17
- not exist yet should not get a dialect string until it has a renderer.
16
+ ``symbol`` is the infix form (``>=``). ``name`` is the stable serialization
17
+ key (``greater_than_or_equals``). SQL is compiled through SQLAlchemy, not
18
+ spelled here.
18
19
  """
19
20
 
20
21
  symbol: str
22
+ name: str
21
23
 
22
24
 
23
25
  class Operator(ValidatingEnum):
@@ -25,16 +27,35 @@ class Operator(ValidatingEnum):
25
27
  A comparison operator, identified by the comparison class that uses it.
26
28
  """
27
29
 
28
- EQUALS = Dialect("==")
29
- NOT_EQUALS = Dialect("!=")
30
- LESS_THAN = Dialect("<")
31
- LESS_THAN_OR_EQUALS = Dialect("<=")
32
- GREATER_THAN = Dialect(">")
33
- GREATER_THAN_OR_EQUALS = Dialect(">=")
34
- IS_IN = Dialect("is in")
35
- CONTAINS = Dialect("contains")
36
- ICONTAINS = Dialect("icontains")
30
+ EQUALS = Dialect("==", "equals")
31
+ NOT_EQUALS = Dialect("!=", "not_equals")
32
+ LESS_THAN = Dialect("<", "less_than")
33
+ LESS_THAN_OR_EQUALS = Dialect("<=", "less_than_or_equals")
34
+ GREATER_THAN = Dialect(">", "greater_than")
35
+ GREATER_THAN_OR_EQUALS = Dialect(">=", "greater_than_or_equals")
36
+ IS_IN = Dialect("is in", "is_in")
37
+ CONTAINS = Dialect("contains", "contains")
38
+ ICONTAINS = Dialect("icontains", "icontains")
37
39
 
38
40
  @property
39
41
  def symbol(self) -> str:
40
42
  return self.value.symbol
43
+
44
+ @property
45
+ def serialized(self) -> str:
46
+ """
47
+ Stable type tag for ``to_dict`` / ``from_dict`` (``Dialect.name``).
48
+
49
+ Named ``serialized`` so it does not shadow ``Enum.name``.
50
+ """
51
+ return self.value.name
52
+
53
+ @classmethod
54
+ def from_serialized(cls, name: str) -> "Operator":
55
+ """
56
+ Resolve an operator from its ``Dialect.name`` / ``serialized`` tag.
57
+ """
58
+ for member in cls:
59
+ if member.serialized == name:
60
+ return member
61
+ raise ValueError(f"Unknown operator serialization {name!r}")
corekit/etl/__init__.py CHANGED
@@ -1,21 +1,9 @@
1
1
  """
2
- ETL pipelines.
2
+ ETL pipelines (legacy).
3
3
 
4
- Implement an extractor, a transformer and a loader, then declare them on an
5
- orchestrator::
6
-
7
- class UserPipeline(BaseETLOrchestrator):
8
- '''
9
- Loads users from the API into the warehouse.
10
- '''
11
- extractor = UserExtractor
12
- transformer = UserTransformer
13
- loader = UserLoader
14
-
15
- await UserPipeline().run()
16
-
17
- The run loop streams, so memory use is bounded by the batch size rather than by
18
- how much data the source holds.
4
+ Prefer ``corekit.integrations`` for new work: ``BaseAdapter`` / ``BaseNormalizer``
5
+ / ``BaseLoader`` / ``BasePipeline``. This package remains for existing tests and
6
+ imports; new vendors should not extend Extractor/Transformer naming.
19
7
  """
20
8
 
21
9
  from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
@@ -0,0 +1,30 @@
1
+ """
2
+ Integration framework: clients, adapters, normalizers, loaders, pipelines.
3
+
4
+ Layering::
5
+
6
+ BaseApiClient — wire + auth to the external system
7
+ BaseNormalizer — raw payload → concrete BaseModel (never dict-out)
8
+ BaseAdapter — named ops (interactive) + optional bulk sources (sync)
9
+ BaseLoader — persist normalized items into the host system
10
+ BasePipeline — stream adapter bulk → loader (nightly / full sync)
11
+
12
+ Interactive Integrations call ``adapter.run(...)``. Catalog syncs call
13
+ ``pipeline.run()``. Do not route point lookups through the pipeline.
14
+ """
15
+
16
+ from corekit.integrations.adapter import BaseAdapter
17
+ from corekit.integrations.loader import BaseLoader
18
+ from corekit.integrations.normalizer import BaseNormalizer
19
+ from corekit.integrations.pipeline import BasePipeline
20
+ from corekit.integrations.schemas import LoadableBatch, NormalizedItem, OperationClass
21
+
22
+ __all__ = [
23
+ "BaseAdapter",
24
+ "BaseLoader",
25
+ "BaseNormalizer",
26
+ "BasePipeline",
27
+ "LoadableBatch",
28
+ "NormalizedItem",
29
+ "OperationClass",
30
+ ]
@@ -0,0 +1,72 @@
1
+ """
2
+ Vendor adapter: named operations plus optional bulk sources for sync pipelines.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+ from collections.abc import AsyncIterable, Callable
9
+ from typing import Any
10
+
11
+ from pydantic import BaseModel
12
+
13
+ from corekit.http import BadRequestException, BaseApiClient
14
+ from corekit.integrations.normalizer import BaseNormalizer
15
+ from corekit.integrations.schemas import NormalizedItem, OperationClass
16
+ from corekit.observability import Loggable
17
+
18
+
19
+ class BaseAdapter(Loggable, ABC):
20
+ """
21
+ Integration surface for one external system.
22
+
23
+ - **Named ops** (``run``): interactive request/response (search, create, …).
24
+ - **Bulk sources** (``iter_bulk``): streaming catalog/sync pulls for pipelines.
25
+
26
+ Always constructed with a ``BaseApiClient`` and a ``BaseNormalizer``. Ops and
27
+ bulk sources return ``BaseModel`` instances — never raw dicts.
28
+ """
29
+
30
+ def __init__(self, client: BaseApiClient, normalizer: BaseNormalizer[Any]) -> None:
31
+ super().__init__()
32
+ self.client = client
33
+ self.normalizer = normalizer
34
+
35
+ @abstractmethod
36
+ def operation_classes(self) -> dict[str, OperationClass]:
37
+ """
38
+ Map operation key → read/write class.
39
+ """
40
+
41
+ @property
42
+ def operations(self) -> frozenset[str]:
43
+ return frozenset(self.operation_classes())
44
+
45
+ def operation_class(self, operation: str) -> OperationClass | None:
46
+ return self.operation_classes().get(operation)
47
+
48
+ def require_operation(self, operation: str) -> OperationClass:
49
+ op_class = self.operation_class(operation)
50
+ if op_class is None:
51
+ raise BadRequestException(detail=f"Unknown operation '{operation}'")
52
+ return op_class
53
+
54
+ @abstractmethod
55
+ async def run(self, operation: str, payload: dict[str, Any], **context: Any) -> BaseModel:
56
+ """
57
+ Execute a named operation. Return a normalized ``BaseModel`` (never secrets).
58
+ """
59
+
60
+ def bulk_sources(self) -> list[Callable[..., AsyncIterable[NormalizedItem]]]:
61
+ """
62
+ Optional streaming sources for full sync. Default: none.
63
+ """
64
+ return []
65
+
66
+ async def iter_bulk(self, *args: Any, **kwargs: Any) -> AsyncIterable[NormalizedItem]:
67
+ """
68
+ Yield normalized items from every bulk source.
69
+ """
70
+ for source in self.bulk_sources():
71
+ async for item in source(*args, **kwargs):
72
+ yield item
@@ -0,0 +1,41 @@
1
+ """
2
+ Persist normalized items into the host system.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+
9
+ from corekit.integrations.schemas import LoadableBatch, NormalizedItem
10
+ from corekit.observability import Loggable
11
+
12
+
13
+ class BaseLoader(Loggable, ABC):
14
+ """
15
+ Batches normalized items and writes them to a destination (DB, namespace, …).
16
+ """
17
+
18
+ def __init__(self) -> None:
19
+ super().__init__()
20
+ self.batch: LoadableBatch = LoadableBatch()
21
+
22
+ def _should_load(self, min_ops: int) -> bool:
23
+ return bool(len(self.batch)) and len(self.batch) >= min_ops
24
+
25
+ def add_item(self, item: NormalizedItem) -> None:
26
+ self.batch.add_item(item)
27
+
28
+ async def flush(self) -> None:
29
+ await self.load(min_ops=0)
30
+
31
+ async def load(self, min_ops: int) -> None:
32
+ if not self._should_load(min_ops):
33
+ return
34
+ await self._load(self.batch)
35
+ self.batch.clear()
36
+
37
+ @abstractmethod
38
+ async def _load(self, batch: LoadableBatch) -> None:
39
+ """
40
+ Write a batch to its destination.
41
+ """
@@ -0,0 +1,28 @@
1
+ """
2
+ Translate raw external payloads into known Pydantic models.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+ from typing import Any, Generic, TypeVar
9
+
10
+ from pydantic import BaseModel
11
+
12
+ from corekit.observability import Loggable
13
+
14
+ TModel = TypeVar("TModel", bound=BaseModel)
15
+
16
+
17
+ class BaseNormalizer(Loggable, ABC, Generic[TModel]):
18
+ """
19
+ Pure translation from vendor/raw payloads to a concrete ``BaseModel``.
20
+
21
+ Output is always structured. Dict-in/dict-out is not a normalizer.
22
+ """
23
+
24
+ @abstractmethod
25
+ def normalize(self, raw: Any) -> TModel:
26
+ """
27
+ Return a typed model downstream code can rely on.
28
+ """
@@ -0,0 +1,136 @@
1
+ """
2
+ Bulk sync: stream adapter bulk sources into a loader.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from dataclasses import dataclass
8
+ from typing import Any, ClassVar
9
+
10
+ from corekit.integrations.adapter import BaseAdapter
11
+ from corekit.integrations.loader import BaseLoader
12
+ from corekit.observability import Loggable
13
+ from corekit.registry import SmartRegistry
14
+
15
+ __all__ = ["BasePipeline"]
16
+
17
+ DEFAULT_BATCH_SIZE = 100
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class PipelineStage:
22
+ """
23
+ One declared stage on a pipeline class.
24
+ """
25
+
26
+ attribute: str
27
+ base_class: type
28
+
29
+ @property
30
+ def label(self) -> str:
31
+ return self.base_class.__name__.removeprefix("Base")
32
+
33
+ def declared_on(self, cls: type) -> Any:
34
+ return cls.__dict__.get(self.attribute)
35
+
36
+ def validate(self, cls: type) -> None:
37
+ stage = getattr(cls, self.attribute, None)
38
+ if stage is None:
39
+ raise TypeError(f"{cls.__name__} is missing its {self.attribute}. Set {self.attribute} = Your{self.label}.")
40
+ if not (isinstance(stage, type) and issubclass(stage, self.base_class)):
41
+ raise TypeError(
42
+ f"{cls.__name__}.{self.attribute} must be a {self.base_class.__name__} subclass, "
43
+ f"got {stage!r}. Assign the class itself, not an instance."
44
+ )
45
+
46
+
47
+ class BasePipeline(Loggable):
48
+ """
49
+ Runs adapter bulk sync into a loader.
50
+
51
+ Interactive named ops stay on ``BaseAdapter.run`` — this class is for
52
+ nightly/full pulls only.
53
+ """
54
+
55
+ adapter: ClassVar[type[BaseAdapter] | None] = None
56
+ loader: ClassVar[type[BaseLoader] | None] = None
57
+ batch_size: ClassVar[int] = DEFAULT_BATCH_SIZE
58
+
59
+ __registry__: SmartRegistry = SmartRegistry()
60
+ __binding_only__: ClassVar[bool] = False
61
+
62
+ STAGES: ClassVar[list[PipelineStage]] = [
63
+ PipelineStage(attribute="adapter", base_class=BaseAdapter),
64
+ PipelineStage(attribute="loader", base_class=BaseLoader),
65
+ ]
66
+
67
+ def __init_subclass__(cls, **kwargs: Any) -> None:
68
+ super().__init_subclass__(**kwargs)
69
+ if cls.__dict__.get("__binding_only__"):
70
+ return
71
+ if not any(stage.declared_on(cls) for stage in cls.STAGES):
72
+ return
73
+ for stage in cls.STAGES:
74
+ stage.validate(cls)
75
+ BasePipeline.__registry__[cls.__name__] = cls
76
+
77
+ def __class_getitem__(cls, item: tuple[type, ...]) -> Any:
78
+ if not isinstance(item, tuple) or len(item) != 2:
79
+ raise TypeError(f"{cls.__name__}[...] takes exactly two stages: {cls.__name__}[Adapter, Loader]")
80
+ bound = {stage.attribute: value for stage, value in zip(cls.STAGES, item, strict=True)}
81
+ return type(f"{cls.__name__}_{item[0].__name__}", (cls,), bound)
82
+
83
+ @classmethod
84
+ def get_pipeline_types(cls) -> SmartRegistry:
85
+ return cls.__registry__
86
+
87
+ @classmethod
88
+ def get_pipeline_by_name(cls, name: str) -> type[BasePipeline] | None:
89
+ return cls.__registry__.get(name)
90
+
91
+ def __init__(
92
+ self,
93
+ adapter: BaseAdapter | None = None,
94
+ loader: BaseLoader | None = None,
95
+ batch_size: int | None = None,
96
+ on_item_error: str = "raise",
97
+ ) -> None:
98
+ if on_item_error not in ("raise", "skip"):
99
+ raise ValueError("on_item_error must be 'raise' or 'skip'")
100
+ super().__init__()
101
+ self.batch_size = batch_size if batch_size is not None else type(self).batch_size
102
+ self.on_item_error = on_item_error
103
+ self._adapter = adapter
104
+ self._loader = loader
105
+
106
+ def _build(self, override: Any, name: str) -> Any:
107
+ if override is not None:
108
+ return override
109
+ stage_class = getattr(type(self), name, None)
110
+ if stage_class is None:
111
+ raise TypeError(f"{type(self).__name__} has no {name}. Declare one, or pass {name}=...")
112
+ return stage_class()
113
+
114
+ async def run(self) -> None:
115
+ """
116
+ Stream ``adapter.iter_bulk()`` into the loader until exhausted.
117
+ """
118
+ adapter = self._build(self._adapter, "adapter")
119
+ loader = self._build(self._loader, "loader")
120
+
121
+ count = 0
122
+ skipped = 0
123
+ async for item in adapter.iter_bulk():
124
+ try:
125
+ loader.add_item(item)
126
+ except Exception:
127
+ if self.on_item_error == "raise":
128
+ raise
129
+ skipped += 1
130
+ self.exception(f"{type(self).__name__} skipped an item")
131
+ continue
132
+ await loader.load(min_ops=self.batch_size)
133
+ count += 1
134
+
135
+ await loader.flush()
136
+ self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")