python-corekit 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

corekit/data/expressions/comparison.py

@@ -1,4 +1,3 @@
-import re
 from typing import Any, Iterable
 
 from corekit.data.expressions.expression import Expression
@@ -18,9 +17,8 @@ class Comparison(Expression):
     """
     Base for comparing two values (Equals, GreaterThan, IsIn, ...).
 
-    `left`, `right`, and `operator` are public so external code -- e.g. a
-    translator to SQL/Mongo/Elasticsearch -- can walk the tree and read out
-    what each node represents without evaluating it. The class itself
+    `left`, `right`, and `operator` are public so a translator can walk the
+    tree and read what each node represents without evaluating it. The class itself
     (Equals, GreaterThan, etc.) is the canonical, type-safe identifier of
     *which* operation a node represents, so a translator can dispatch on
     isinstance(node, GreaterThan) rather than string-matching on symbol.
@@ -73,35 +71,11 @@ class Comparison(Expression):
         """
         return isinstance(self.right, FieldExpression)
 
-    def _reject_field_comparison(self) -> None:
+    def _reject_sql(self, reason: str) -> None:
         """
-        Guard a dialect whose clauses only compare a field to a literal.
-
-        Rendering the right side would emit a field's *name* where a value was
-        meant, which is a silently wrong query rather than a failed one.
-        """
-        if self.compares_fields:
-            raise NotImplementedError(f"Elasticsearch cannot express `{self.symbol}` between two fields")
-
-    def _reference(self, side: Any) -> str:
-        """
-        One side as a Mongo field reference, e.g. ``$age``.
+        Refuse a form SQL cannot mean the same way Python does.
         """
-        return f"${self.resolve(side, target=Target.NAME)}"
-
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        if self.compares_fields:
-            return {"$expr": {self.operator.mongo: [self._reference(self.left), self._reference(self.right)]}}
-        return {self.get_left(target=Target.NAME): {self.operator.mongo: self.get_right()}}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        self._reject_field_comparison()
-        # Elasticsearch range operators are the Mongo spelling without the `$`.
-        return {
-            self.operator.elasticsearch: {
-                self.get_left(target=Target.NAME): {self.operator.mongo.lstrip("$"): self.get_right()}
-            }
-        }
+        raise NotImplementedError(f"SQL cannot express `{self.symbol}`: {reason}")
 
 
 class Equals(Comparison):
@@ -110,16 +84,6 @@ class Equals(Comparison):
     def __call__(self, context: Any, *args: Any, **kwargs: Any) -> bool:
         return self.get_left(context, *args, **kwargs) == self.get_right(context, *args, **kwargs)
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        if self.compares_fields:
-            return super().to_mongo(*args, **kwargs)
-        # Equality is the one operator Mongo spells as a bare value.
-        return {self.get_left(target=Target.NAME): self.get_right()}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        self._reject_field_comparison()
-        return {self.operator.elasticsearch: {self.get_left(target=Target.NAME): self.get_right()}}
-
     def to_sqlalchemy(self, context: Any) -> Any:
         return self.get_left(context) == self.get_right(context)
 
@@ -130,14 +94,15 @@ class NotEquals(Comparison):
     def __call__(self, context: Any, *args: Any, **kwargs: Any) -> bool:
         return self.get_left(context, *args, **kwargs) != self.get_right(context, *args, **kwargs)
 
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        self._reject_field_comparison()
-        return {
-            self.operator.elasticsearch: {self.get_left(target=Target.NAME): {self.operator.mongo: self.get_right()}}
-        }
-
     def to_sqlalchemy(self, context: Any) -> Any:
-        return self.get_left(context) != self.get_right(context)
+        left = self.get_left(context)
+        # Python treats None != value as True. SQL treats NULL != value as
+        # unknown and drops the row, so a null column must be included.
+        if self.right is None:
+            return left.is_not(None)
+        if isinstance(self.right, FieldExpression):
+            return left != self.get_right(context)
+        return (left != self.get_right(context)) | left.is_(None)
 
 
 class LessThan(Comparison):
@@ -186,17 +151,9 @@ class IsIn(Comparison):
     def __call__(self, context: Any, *args: Any, **kwargs: Any) -> bool:
         return self.get_left(context, *args, **kwargs) in self.get_right(context, *args, **kwargs)
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        if self.compares_fields:
-            return {"$expr": {self.operator.mongo: [self._reference(self.left), self._reference(self.right)]}}
-        # A list, not the stored frozenset: no driver can serialize a set.
-        return {self.get_left(target=Target.NAME): {self.operator.mongo: self._values()}}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        self._reject_field_comparison()
-        return {self.operator.elasticsearch: {self.get_left(target=Target.NAME): self._values()}}
-
     def to_sqlalchemy(self, context: Any) -> Any:
+        if self.compares_fields:
+            self._reject_sql("the right side is a field, and SQL IN needs a list of values")
         return self.get_left(context).in_(self._values(context))
 
     def _values(self, context: Any = None) -> list[Any]:
@@ -219,21 +176,10 @@ class Contains(Comparison):
     def __call__(self, context: Any, *args: Any, **kwargs: Any) -> bool:
         return self.get_right(context, *args, **kwargs) in self.get_left(context, *args, **kwargs)
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        if self.compares_fields:
-            # $regex is a query operator, not an aggregation one, so it cannot
-            # appear inside the $expr form that comparing two fields needs.
-            raise NotImplementedError("Mongo cannot express `contains` between two fields")
-        # $regex with the operand escaped, so a value containing regex
-        # metacharacters is matched literally rather than as a pattern.
-        return {self.get_left(target=Target.NAME): {self.operator.mongo: re.escape(str(self.get_right()))}}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        self._reject_field_comparison()
-        return {self.operator.elasticsearch: {self.get_left(target=Target.NAME): f"*{self.get_right()}*"}}
-
     def to_sqlalchemy(self, context: Any) -> Any:
-        return self.get_left(context).contains(self.get_right(context))
+        # Python is case-sensitive `in`. SQL LIKE is case-insensitive for ASCII
+        # on SQLite, so compiling this would match a different set of rows.
+        self._reject_sql("substring match is case-sensitive in Python and LIKE is not")
 
 
 class ValueExpression(Expression):
@@ -241,8 +187,7 @@ class ValueExpression(Expression):
     Expression representing a single value to be compared.
 
     The resulting tree (Equals/And/Or/... nodes) is also an inspectable structure,
-    not just a callable. These could be used in the future to emit structured queries
-    (SQL, MongoDB, Elasticsearch, etc.)
+    not just a callable. SQL is emitted through ``to_sqlalchemy``.
     """
 
     def __init__(self, value: Any) -> None:
@@ -284,12 +229,6 @@ class ValueExpression(Expression):
         """
         return self.value
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return self.value
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return self.value
-
 
 class FieldExpression(ValueExpression):
     """

corekit/data/expressions/expression.py

@@ -18,12 +18,6 @@ class Expression:
     def __invert__(self) -> "Expression":
         return Not(self)
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        raise NotImplementedError
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        raise NotImplementedError
-
     def to_sqlalchemy(self, context: Any) -> Any:
         """
         Compile to a SQLAlchemy clause against ``context``, a model class.
@@ -44,16 +38,6 @@ class And(Expression):
     def __repr__(self) -> str:
         return f"({self.left!r} & {self.right!r})"
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {"$and": [self.left.to_mongo(*args, **kwargs), self.right.to_mongo(*args, **kwargs)]}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {
-            "bool": {
-                "must": [self.left.to_elasticsearch(*args, **kwargs), self.right.to_elasticsearch(*args, **kwargs)]
-            }
-        }
-
     def to_sqlalchemy(self, context: Any) -> Any:
         return self.left.to_sqlalchemy(context) & self.right.to_sqlalchemy(context)
 
@@ -69,16 +53,6 @@ class Or(Expression):
     def __repr__(self) -> str:
         return f"({self.left!r} | {self.right!r})"
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {"$or": [self.left.to_mongo(*args, **kwargs), self.right.to_mongo(*args, **kwargs)]}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {
-            "bool": {
-                "should": [self.left.to_elasticsearch(*args, **kwargs), self.right.to_elasticsearch(*args, **kwargs)]
-            }
-        }
-
     def to_sqlalchemy(self, context: Any) -> Any:
         return self.left.to_sqlalchemy(context) | self.right.to_sqlalchemy(context)
 
@@ -93,11 +67,5 @@ class Not(Expression):
     def __repr__(self) -> str:
         return f"~{self.expr!r}"
 
-    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {"$not": self.expr.to_mongo(*args, **kwargs)}
-
-    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
-        return {"bool": {"must_not": self.expr.to_elasticsearch(*args, **kwargs)}}
-
     def to_sqlalchemy(self, context: Any) -> Any:
         return ~self.expr.to_sqlalchemy(context)

corekit/data/expressions/operator.py

@@ -1,5 +1,5 @@
 """
-The operators a comparison can express, and how each dialect spells them.
+The operators a comparison can express.
 """
 
 from typing import NamedTuple
@@ -11,44 +11,29 @@ __all__ = ["Dialect", "Operator"]
 
 class Dialect(NamedTuple):
     """
-    How one operator is spelled in each backend.
+    How one operator is spelled for humans.
 
-    Args:
-        symbol: The human-readable operator, used in ``repr``.
-        mongo: The MongoDB query operator.
-        elasticsearch: The Elasticsearch clause the operator renders into.
+    SQL is compiled through SQLAlchemy, not spelled here. A backend that does
+    not exist yet should not get a dialect string until it has a renderer.
     """
 
     symbol: str
-    mongo: str
-    elasticsearch: str
 
 
 class Operator(ValidatingEnum):
     """
-    A comparison operator and its spelling in every dialect corekit renders to.
-
-    Adding a backend means adding a field to ``Dialect``, not a method to
-    every comparison class.
+    A comparison operator, identified by the comparison class that uses it.
     """
 
-    EQUALS = Dialect("==", "$eq", "term")
-    NOT_EQUALS = Dialect("!=", "$ne", "term")
-    LESS_THAN = Dialect("<", "$lt", "range")
-    LESS_THAN_OR_EQUALS = Dialect("<=", "$lte", "range")
-    GREATER_THAN = Dialect(">", "$gt", "range")
-    GREATER_THAN_OR_EQUALS = Dialect(">=", "$gte", "range")
-    IS_IN = Dialect("is in", "$in", "terms")
-    CONTAINS = Dialect("contains", "$regex", "wildcard")
+    EQUALS = Dialect("==")
+    NOT_EQUALS = Dialect("!=")
+    LESS_THAN = Dialect("<")
+    LESS_THAN_OR_EQUALS = Dialect("<=")
+    GREATER_THAN = Dialect(">")
+    GREATER_THAN_OR_EQUALS = Dialect(">=")
+    IS_IN = Dialect("is in")
+    CONTAINS = Dialect("contains")
 
     @property
     def symbol(self) -> str:
         return self.value.symbol
-
-    @property
-    def mongo(self) -> str:
-        return self.value.mongo
-
-    @property
-    def elasticsearch(self) -> str:
-        return self.value.elasticsearch

corekit/data/stats.py

@@ -88,6 +88,9 @@ class FieldDescription:
 
     @property
     def is_numeric(self) -> bool:
+        # all([]) is True, which would then call statistics.fmean([]) and crash.
+        if not self.non_missing:
+            return False
         return all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in self.non_missing)
 
     def to_field_stats(self) -> FieldStats:

corekit/decorators/exception_handling.py

@@ -1,4 +1,5 @@
 import functools
+import inspect
 import logging
 from typing import Any, Callable
 
@@ -8,6 +9,19 @@ from corekit.utils.coercion import safe_tuple
 logger = logging.getLogger(__name__)
 
 
+def _log_text(ex: Exception) -> str:
+    """
+    Prefer a backend log line when the exception provides one.
+
+    Public exceptions stringify to the user-facing copy. Logging that drops
+    the internal message.
+    """
+    for_log = getattr(ex, "for_log", None)
+    if callable(for_log):
+        return for_log()
+    return str(ex)
+
+
 def exception_handler(
     ignore: list[type[Exception]] | None = None,
     callback: Callable = raise_exc,
@@ -16,6 +30,9 @@ def exception_handler(
     """
     A decorator to handle exceptions in a class method or function.
 
+    Works on both sync and async functions. An async function keeps an async
+    wrapper, so awaiting it still works.
+
     Args:
         ignore: exception classes to suppress. These do not propagate.
         callback: called when an exception is raised that is not ignored.
@@ -24,21 +41,32 @@ def exception_handler(
     exceptions_to_ignore = safe_tuple(ignore)
 
     def decorator(func):
+        def handle(ex: Exception, args: tuple, kwargs: dict) -> Any:
+            text = _log_text(ex)
+            if isinstance(ex, exceptions_to_ignore):
+                logger.warning(f"Suppressed exception in {func.__name__}: {text}")
+                logger.info(f"Invoking callback function: {ignore_callback.__name__}")
+                return ignore_callback(ex, *args, **kwargs)
+
+            logger.error(f"Exception in {func.__name__}: {text}", exc_info=True)
+            return callback(ex, *args, **kwargs)
+
         @functools.wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             try:
                 return func(*args, **kwargs)
             except Exception as ex:
-                # If in ignore list, suppress and invoke the ignore_callback function
-                if isinstance(ex, exceptions_to_ignore):
-                    logger.warning(f"Suppressed exception in {func.__name__}: {ex}")
-                    logger.info(f"Invoking callback function: {ignore_callback.__name__}")
-                    return ignore_callback(ex, *args, **kwargs)
+                return handle(ex, args, kwargs)
 
-                # Log the exception and invoke the callback function, returning its return value (if any)
-                logger.error(f"Exception in {func.__name__}: {ex}", exc_info=True)
-                return callback(ex, *args, **kwargs)
+        @functools.wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return await func(*args, **kwargs)
+            except Exception as ex:
+                return handle(ex, args, kwargs)
 
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
         return wrapper
 
     return decorator

corekit/docker/watchdog.py

@@ -42,11 +42,12 @@ class Watchdog(Benchmarkable):
     background daemon, and one unreachable container should not stop it.
     """
 
-    def __init__(self, docker_host: str | None = None, enforce_label: bool = False) -> None:
+    def __init__(self, docker_host: str | None = None, enforce_label: bool = True) -> None:
         """
         :param docker_host: Docker endpoint. Defaults to the local socket.
         :param enforce_label: restrict actions to containers carrying the
-            managed label.
+            managed label. On by default; turn it off only when this watchdog
+            is meant to control every container on the host.
         """
         super().__init__()
         self.docker_host = docker_host or DEFAULT_DOCKER_HOST
@@ -96,27 +97,29 @@ class Watchdog(Benchmarkable):
             return container.labels.get(WATCHDOG_LABEL) == WATCHDOG_MANAGED
         return True
 
-    def _start_container(self, container: Container) -> None:
+    def _start_container(self, container: Container) -> bool:
         try:
             container.reload()
             if container.status != "running" and self._label_validator(container):
                 container.start()
                 self.info(f"Started container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is already running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is already running or not managed.")
         except APIError as exc:
             self.error(f"Failed to start container {container.name}: {exc}")
+        return False
 
-    def _stop_container(self, container: Container) -> None:
+    def _stop_container(self, container: Container) -> bool:
         try:
             container.reload()
             if container.status == "running" and self._label_validator(container):
                 container.stop()
                 self.info(f"Stopped container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not running or not managed.")
         except APIError as exc:
             self.error(f"Failed to stop container {container.name}: {exc}")
+        return False
 
     def _by_name(self, name: str) -> Container | None:
         """
@@ -161,62 +164,78 @@ class Watchdog(Benchmarkable):
         for container in self._container_iterator(filter_func=self._get_label_filter(label, value), all=True):
             self._start_container(container)
 
-    def start_container_by_name(self, name: str) -> None:
+    def start_container_by_name(self, name: str) -> bool:
         """
-        Start one container by name.
+        Start one container by name. Returns whether it was started.
         """
         container = self._by_name(name)
-        if container is not None:
-            self._start_container(container)
+        if container is None:
+            return False
+        return self._start_container(container)
 
-    def stop_container_by_name(self, name: str) -> None:
+    def stop_container_by_name(self, name: str) -> bool:
         """
-        Stop one container by name.
+        Stop one container by name. Returns whether it was stopped.
         """
         container = self._by_name(name)
-        if container is not None:
-            self._stop_container(container)
+        if container is None:
+            return False
+        return self._stop_container(container)
 
-    def restart_container_by_name(self, name: str) -> None:
+    def restart_container_by_name(self, name: str) -> bool:
         """
-        Stop and then start one container by name.
+        Restart one container by name, honoring the managed label.
+
+        Returns whether the container was restarted. A missing or unmanaged
+        container is not touched.
         """
         container = self._by_name(name)
         if container is None:
-            return
-        self._stop_container(container)
-        self._start_container(container)
+            return False
+        try:
+            container.reload()
+            if not self._label_validator(container):
+                self.info(f"Container {container.name!r} is not managed.")
+                return False
+            container.restart()
+            self.info(f"Restarted container: {container.name}")
+            return True
+        except APIError as exc:
+            self.error(f"Failed to restart container {name}: {exc}")
+            return False
 
-    def pause_container_by_name(self, name: str) -> None:
+    def pause_container_by_name(self, name: str) -> bool:
         """
-        Pause one running container by name.
+        Pause one running container by name. Returns whether it was paused.
         """
         container = self._by_name(name)
         if container is None:
-            return
+            return False
         try:
             container.reload()
             if container.status == "running" and self._label_validator(container):
                 container.pause()
                 self.info(f"Paused container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not running or not managed.")
         except APIError as exc:
             self.error(f"Failed to pause container {name}: {exc}")
+        return False
 
-    def unpause_container_by_name(self, name: str) -> None:
+    def unpause_container_by_name(self, name: str) -> bool:
         """
-        Unpause one paused container by name.
+        Unpause one paused container by name. Returns whether it was unpaused.
         """
         container = self._by_name(name)
         if container is None:
-            return
+            return False
         try:
             container.reload()
             if container.status == "paused" and self._label_validator(container):
                 container.unpause()
                 self.info(f"Unpaused container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not paused or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not paused or not managed.")
         except APIError as exc:
             self.error(f"Failed to unpause container {name}: {exc}")
+        return False

corekit/etl/__init__.py

@@ -18,7 +18,7 @@ The run loop streams, so memory use is bounded by the batch size rather than by
 how much data the source holds.
 """
 
-from corekit.etl.connection import BaseConnection, ConnectionDetails
+from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
 from corekit.etl.extract.extractor import BaseETLExtractor
 from corekit.etl.extract.schemas import BaseExtractedItemModel, ExtractedItem
 from corekit.etl.load.loader import BaseETLLoader
@@ -30,6 +30,7 @@ from corekit.etl.transform.transformer import BaseETLTransformer
 
 __all__ = [
     "BaseConnection",
+    "ETLSource",
     "BaseETLExtractor",
     "BaseETLLoader",
     "BaseETLOrchestrator",

corekit/etl/connection.py

@@ -7,24 +7,21 @@ class ConnectionDetails(NamedTuple):
     description: str
 
 
-class BaseConnection(Enum):
+class ETLSource(Enum):
     """
-    Base enum for ETL Connections. All ETL connections must inherit from this class.
+    A named source label for an extracted item.
+
+    This is not a connection. It does not open a socket, a session, or a
+    ``Connectable``. It only names where a row came from so a transformer can
+    tell sources apart.
 
     Example:
-        class MyConnectionEnum(BaseConnection):
-            MY_FIRST_CONNECTION = ConnectionDetails(
-                name="my_first_connection",
-                description="My First Connection"
-            )
-            ANOTHER_CONNECTION = ConnectionDetails(
-                name="another_connection",
-                description="Another Connection"
-            )
+        class UserSource(ETLSource):
+            USERS = ConnectionDetails(name="users", description="User records")
     """
 
     @classmethod
-    def from_name(cls, name: str) -> "BaseConnection":
+    def from_name(cls, name: str) -> "ETLSource":
         # TODO: Make sure this is efficient
         for connection in cls:
             if connection.value.name == name:
@@ -42,3 +39,8 @@ class BaseConnection(Enum):
         Helper method for fetching the name of the connection
         """
         return self.value.description
+
+
+# The old name implied a second connection system. Keep it so existing imports
+# still resolve; new code should use ETLSource.
+BaseConnection = ETLSource

corekit/etl/extract/extractor.py

@@ -1,30 +1,23 @@
 from abc import ABC, abstractmethod
 from typing import Any, AsyncIterable, Callable
 
-from corekit.etl.connection import BaseConnection
 from corekit.etl.extract.schemas import ExtractedItem
-from corekit.http.client import BaseApiClient
 from corekit.observability.loggable import Loggable
 
 
 class BaseETLExtractor(Loggable, ABC):
     """
-    Base class for ETL extractors.
+    Yields items from a source.
+
+    The runner constructs this with no arguments, so a subclass that needs a
+    client or other dependency should default it or be passed in as an
+    instance. ``client`` is optional and unused by the base class.
     """
 
-    def __init__(self, client: BaseApiClient) -> None:
+    def __init__(self, client: Any | None = None) -> None:
         super().__init__()
         self.client = client
 
-    @property
-    @abstractmethod
-    def connection(self) -> BaseConnection:
-        """
-        Abstract property that must be implemented by child classes.
-        This property should return the integration object that the extractor is associated with
-        """
-        raise NotImplementedError
-
     @abstractmethod
     def _extraction_methods(self) -> list[Callable[[Any], Any]]:
         """

corekit/etl/orchestrator.py

@@ -159,12 +159,19 @@ class BaseETLOrchestrator(Loggable):
         transformer: BaseETLTransformer | None = None,
         loader: BaseETLLoader | None = None,
         batch_size: int | None = None,
+        on_item_error: str = "raise",
     ) -> None:
         """
         Build a pipeline, optionally overriding any stage with an instance.
+
+        ``on_item_error`` is ``raise`` (stop, leaving earlier batches written)
+        or ``skip`` (log the item and continue). Transform is synchronous.
         """
+        if on_item_error not in ("raise", "skip"):
+            raise ValueError("on_item_error must be 'raise' or 'skip'")
         super().__init__()
         self.batch_size = batch_size if batch_size is not None else type(self).batch_size
+        self.on_item_error = on_item_error
         self._extractor = extractor
         self._transformer = transformer
         self._loader = loader
@@ -192,10 +199,20 @@ class BaseETLOrchestrator(Loggable):
         loader = self._build(self._loader, "loader")
 
         count = 0
+        skipped = 0
         async for extracted_item in extractor.extract():
-            loader.add_item(transformer.transform(extracted_item))
+            try:
+                # transform is synchronous; do not await it.
+                transformed = transformer.transform(extracted_item)
+            except Exception:
+                if self.on_item_error == "raise":
+                    raise
+                skipped += 1
+                self.exception(f"{type(self).__name__} skipped an item")
+                continue
+            loader.add_item(transformed)
             await loader.load(min_ops=self.batch_size)
             count += 1
 
         await loader.flush()
-        self.info(f"{type(self).__name__} processed {count} items")
+        self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")