loom-kernel 0.9.2__tar.gz → 0.10.0__tar.gz

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/CHANGELOG.md

@@ -1,3 +1,37 @@
+# 🚀 Release 0.10.0 ([#41](https://github.com/the-reacher-data/loom-py/pull/41)) ([`099de38`](https://github.com/the-reacher-data/loom-py/commit/099de38b47cd7002a9c36fc56c9c85f673431022))
+
+
+## ✨ Features
+### prefect
+- **prefect:** add loom.prefect observer module for ETL observability<br>
+  > Introduces loom.prefect as an optional integration layer that exposes each
+  > ETLStep as an observable Prefect @task, giving Prefect full step-level
+  > visibility into ETL pipelines rather than treating the whole run as a
+  > single opaque process.
+  > Key additions:
+  > FlowCtx: typed operational contract (correlation_id, run_id, processes, force, dry_run)
+  > RunManifest + ManifestStore: ephemeral S3-backed retry state — zero S3 writes on happy path
+  > S3JsonManifestStore: fsspec-backed implementation with correlation_id path validation
+  > PrefectObserver: LifecycleObserver forwarding loom events to Prefect UI logs/artifacts
+  > FlowConfig + _load_flow_config: per-flow retry policy from YAML
+  > build_etl_flow(): factory that compiles a PipelinePlan once at build time,
+  > creates the runner once per Fargate container, and submits each step as @task
+  > respecting ParallelStepGroup topology and skipping SUCCESS steps from the manifest
+  > ETLRunner.run() now accepts optional run_id so Prefect and loom lineage share
+  > the same traceability identifier
+
+- **prefect:** add Fargate/Docker launcher + deploy_etl() for ETL flow generation
+
+
+
+
+
+
+
+
+
+
+
 # 🚀 Release 0.9.2 ([#39](https://github.com/the-reacher-data/loom-py/pull/39)) ([`f1a482d`](https://github.com/the-reacher-data/loom-py/commit/f1a482d03f8b83f8e0c7725e7af6160466ac6126))
 
 

loom_kernel-0.10.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,32 @@
+# 🚀 Release 0.10.0 ([#41](https://github.com/the-reacher-data/loom-py/pull/41)) ([`099de38`](https://github.com/the-reacher-data/loom-py/commit/099de38b47cd7002a9c36fc56c9c85f673431022))
+
+
+## ✨ Features
+### prefect
+- **prefect:** add loom.prefect observer module for ETL observability<br>
+  > Introduces loom.prefect as an optional integration layer that exposes each
+  > ETLStep as an observable Prefect @task, giving Prefect full step-level
+  > visibility into ETL pipelines rather than treating the whole run as a
+  > single opaque process.
+  > Key additions:
+  > FlowCtx: typed operational contract (correlation_id, run_id, processes, force, dry_run)
+  > RunManifest + ManifestStore: ephemeral S3-backed retry state — zero S3 writes on happy path
+  > S3JsonManifestStore: fsspec-backed implementation with correlation_id path validation
+  > PrefectObserver: LifecycleObserver forwarding loom events to Prefect UI logs/artifacts
+  > FlowConfig + _load_flow_config: per-flow retry policy from YAML
+  > build_etl_flow(): factory that compiles a PipelinePlan once at build time,
+  > creates the runner once per Fargate container, and submits each step as @task
+  > respecting ParallelStepGroup topology and skipping SUCCESS steps from the manifest
+  > ETLRunner.run() now accepts optional run_id so Prefect and loom lineage share
+  > the same traceability identifier
+
+- **prefect:** add Fargate/Docker launcher + deploy_etl() for ETL flow generation
+
+
+
+
+
+
+
+
+

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loom-kernel
-Version: 0.9.2
+Version: 0.10.0
 Summary: Loom Python project
 License-File: LICENSE
 Classifier: Typing :: Typed
@@ -50,6 +50,13 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-spark'
 Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-spark'
 Requires-Dist: pyspark<4.0,>=3.5; extra == 'etl-spark'
+Provides-Extra: prefect
+Requires-Dist: boto3>=1.35; extra == 'prefect'
+Requires-Dist: docker>=7.0; extra == 'prefect'
+Requires-Dist: fsspec>=2024.2.0; extra == 'prefect'
+Requires-Dist: omegaconf<3.0,>=2.3; extra == 'prefect'
+Requires-Dist: prefect<4.0,>=3.0; extra == 'prefect'
+Requires-Dist: pyyaml>=6.0; extra == 'prefect'
 Provides-Extra: prometheus
 Requires-Dist: prometheus-client>=0.20; extra == 'prometheus'
 Provides-Extra: pyspark

loom_kernel-0.10.0/docker-compose.local.yaml

@@ -0,0 +1,43 @@
+# Minimal PoC stack for the Prefect → Fargate/Docker launcher.
+# Brings up a local Prefect Server plus a process worker that spawns ETL
+# containers via the host's docker socket.
+#
+# ⚠️  SECURITY WARNING
+# The prefect-worker mounts /var/run/docker.sock. Any process inside that
+# container can spawn arbitrary containers on the host, mount the host
+# filesystem, and escalate to host root. This compose file is for LOCAL
+# DEVELOPMENT ONLY. Never deploy it to a shared / production environment.
+#
+# Usage:
+#   docker compose -f docker-compose.local.yaml up -d
+#   docker build -t loom-etl-daily-orders:dev -f Dockerfile.daily_orders .
+#   python -m my_app.deploy
+#   prefect deployment run "daily-orders/daily-orders" -p env=local
+
+networks:
+  loom_net:
+    driver: bridge
+
+services:
+  prefect-server:
+    image: prefecthq/prefect:3-latest
+    command: prefect server start --host 0.0.0.0
+    ports:
+      - "4200:4200"
+    networks: [loom_net]
+    environment:
+      PREFECT_UI_API_URL: http://localhost:4200/api
+
+  prefect-worker:
+    image: prefecthq/prefect:3-latest
+    command: >
+      sh -c "prefect config set PREFECT_API_URL=http://prefect-server:4200/api &&
+             prefect worker start --pool default --type process"
+    depends_on: [prefect-server]
+    networks: [loom_net]
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./:/workspace
+    working_dir: /workspace
+    environment:
+      PREFECT_API_URL: http://prefect-server:4200/api

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "loom-kernel"
-version = "0.9.2"
+version = "0.10.0"
 description = "Loom Python project"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -70,6 +70,14 @@ etl-otel = [
     "opentelemetry-exporter-otlp-proto-http>=1.20,<2.0",
     "opentelemetry-exporter-otlp-proto-grpc>=1.20,<2.0",
 ]
+prefect = [
+    "prefect>=3.0,<4.0",
+    "fsspec>=2024.2.0",
+    "pyyaml>=6.0",
+    "omegaconf>=2.3,<3.0",
+    "boto3>=1.35",
+    "docker>=7.0",
+]
 pyspark = [
     "pyspark>=3.5,<4.0",
 ]
@@ -127,7 +135,6 @@ dev = [
     "pyarrow>=14.0",
     "deltalake>=1.5,<2.0",
     "fsspec>=2024.2.0",
-    "httpx>=0.28.1",
     "httptools>=0.6,<1.0",
     "uvloop>=0.21,<1.0; sys_platform != 'win32'",
     "prometheus-client>=0.20",
@@ -143,6 +150,9 @@ dev = [
     "opentelemetry-sdk>=1.20,<2.0",
     "opentelemetry-exporter-otlp-proto-http>=1.20,<2.0",
     "opentelemetry-exporter-otlp-proto-grpc>=1.20,<2.0",
+    "prefect>=3.0,<4.0",
+    "pyyaml>=6.0",
+    "httpx>=0.27",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -224,6 +234,17 @@ module = [
 ignore_missing_imports = true
 follow_imports = "skip"
 
+[[tool.mypy.overrides]]
+module = [
+    "docker",
+    "docker.*",
+    "requests",
+    "requests.*",
+    "prefect",
+    "prefect.*",
+]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 minversion = "8.0"
 testpaths = [
@@ -247,7 +268,7 @@ markers = [
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.9.2"
+version = "0.10.0"
 tag_format = "v$version"
 version_files = [
     "pyproject.toml:project.version",

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/sonar-project.properties

@@ -24,7 +24,7 @@ sonar.coverage.exclusions=tests/**,tests/**/*,**/.github/**,docs/**,docs/**/*
 #                   String(), DateTime(), Numeric()
 #
 # Renaming any of them would be a breaking public API change.
-sonar.issue.ignore.multicriteria=e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15,e16,e17,e18,e19,e20,e21,e22
+sonar.issue.ignore.multicriteria=e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15,e16,e17,e18,e19,e20,e21,e22,e23,e24,e25,e26
 
 # Use-case DSL markers: Input, Load, LoadById, Exists — S100 (method) + S1542 (function)
 sonar.issue.ignore.multicriteria.e1.ruleKey=python:S100
@@ -126,6 +126,22 @@ sonar.issue.ignore.multicriteria.e21.resourceKey=src/loom/celery/service.py
 sonar.issue.ignore.multicriteria.e22.ruleKey=python:S1172
 sonar.issue.ignore.multicriteria.e22.resourceKey=src/loom/celery/service.py
 
+# python:S2068 / python:S6437 — "hard-coded credentials" / "hard-coded secrets".
+# _polars.py defines a static mapping from fsspec storage-option KEY NAMES
+# ("endpoint_url", "key", "secret", "token") to their Rust ``object_store``
+# equivalents. These strings are public fsspec/object_store field identifiers,
+# never credential values. Renaming them would break compatibility with
+# fsspec's documented storage_options API.
+sonar.issue.ignore.multicriteria.e23.ruleKey=python:S2068
+sonar.issue.ignore.multicriteria.e23.resourceKey=src/loom/etl/checkpoint/_backends/_polars.py
+sonar.issue.ignore.multicriteria.e24.ruleKey=python:S6437
+sonar.issue.ignore.multicriteria.e24.resourceKey=src/loom/etl/checkpoint/_backends/_polars.py
+# Alternate rule keys some Sonar setups use for the same hard-coded-secret family.
+sonar.issue.ignore.multicriteria.e25.ruleKey=python:S6418
+sonar.issue.ignore.multicriteria.e25.resourceKey=src/loom/etl/checkpoint/_backends/_polars.py
+sonar.issue.ignore.multicriteria.e26.ruleKey=secrets:*
+sonar.issue.ignore.multicriteria.e26.resourceKey=src/loom/etl/checkpoint/_backends/_polars.py
+
 sonar.sourceEncoding=UTF-8
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.python.xunit.reportPath=junit.xml

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/src/loom/core/logger/config.py

@@ -81,9 +81,13 @@ class StreamHandlerConfig(LoomFrozenStruct, frozen=True, tag="stream", tag_field
 
     Attributes:
         stream: Target stream — ``"stdout"`` or ``"stderr"``.
+        renderer: Per-handler output renderer (``"console"`` / ``"json"``).
+            When ``None`` the handler inherits the top-level
+            :attr:`LogConfig.renderer`.
     """
 
     stream: Literal["stdout", "stderr"] = "stdout"
+    renderer: str | None = None
 
 
 class FileHandlerConfig(LoomFrozenStruct, frozen=True, tag="file", tag_field="type"):
@@ -92,10 +96,12 @@ class FileHandlerConfig(LoomFrozenStruct, frozen=True, tag="file", tag_field="ty
     Attributes:
         filename: Absolute or relative path to the log file.
         encoding: File encoding. Defaults to ``"utf-8"``.
+        renderer: Per-handler output renderer. See :class:`StreamHandlerConfig`.
     """
 
     filename: str
     encoding: str = "utf-8"
+    renderer: str | None = None
 
 
 class RotatingFileHandlerConfig(
@@ -108,12 +114,14 @@ class RotatingFileHandlerConfig(
         max_bytes: Maximum file size in bytes before rotation. Defaults to 10 MB.
         backup_count: Number of rotated backup files to retain.
         encoding: File encoding. Defaults to ``"utf-8"``.
+        renderer: Per-handler output renderer. See :class:`StreamHandlerConfig`.
     """
 
     filename: str
     max_bytes: int = 10 * 1024 * 1024
     backup_count: int = 5
     encoding: str = "utf-8"
+    renderer: str | None = None
 
 
 HandlerConfig = StreamHandlerConfig | FileHandlerConfig | RotatingFileHandlerConfig
@@ -217,11 +225,31 @@ def _parse_level(level: str) -> int:
     return value
 
 
-def _resolve_renderer(config: LogConfig) -> Any:
+def _default_renderer(config: LogConfig) -> Renderer:
     is_prod = config.environment == Environment.PROD
-    renderer = config.renderer or (Renderer.JSON if is_prod else Renderer.CONSOLE)
-    colors = config.colors if config.colors is not None else not is_prod
-    return _RENDERER_FACTORIES[renderer](colors)
+    return config.renderer or (Renderer.JSON if is_prod else Renderer.CONSOLE)
+
+
+def _default_colors(config: LogConfig) -> bool:
+    if config.colors is not None:
+        return config.colors
+    return config.environment != Environment.PROD
+
+
+def _resolve_renderer(config: LogConfig) -> Any:
+    """Top-level renderer (used by ``_setup_structlog`` for the wrapper chain)."""
+    return _RENDERER_FACTORIES[_default_renderer(config)](_default_colors(config))
+
+
+def _handler_renderer(handler_cfg: HandlerConfig, config: LogConfig) -> Any:
+    """Build the structlog renderer instance for a single handler.
+
+    Honours the per-handler ``renderer`` field when set; otherwise falls
+    back to the top-level :attr:`LogConfig.renderer` / environment default.
+    """
+    raw = getattr(handler_cfg, "renderer", None)
+    renderer = Renderer.from_str(raw) if raw is not None else _default_renderer(config)
+    return _RENDERER_FACTORIES[renderer](_default_colors(config))
 
 
 def _build_stdlib_handler(cfg: HandlerConfig) -> logging.Handler:
@@ -244,18 +272,40 @@ def _build_stdlib_handler(cfg: HandlerConfig) -> logging.Handler:
 
 
 def _setup_stdlib(config: LogConfig, level: int) -> None:
-    if not config.handlers:
-        logging.basicConfig(level=level, format="%(message)s", force=True)
-        return
-
+    # Every stdlib record (loom-internal or third-party) is routed through
+    # a structlog ProcessorFormatter so the whole process shares one render.
     target = logging.getLogger(config.name) if config.name else logging.getLogger()
     target.setLevel(level)
-    for handler_cfg in config.handlers:
+    while target.handlers:
+        target.removeHandler(target.handlers[-1])
+
+    for handler_cfg in config.handlers or (StreamHandlerConfig(),):
         handler = _build_stdlib_handler(handler_cfg)
         handler.setLevel(level)
+        handler.setFormatter(
+            structlog.stdlib.ProcessorFormatter(
+                processor=_handler_renderer(handler_cfg, config),
+                foreign_pre_chain=_foreign_pre_chain(),
+            )
+        )
         target.addHandler(handler)
 
 
+def _foreign_pre_chain() -> list[Any]:
+    """Pre-chain applied to records that did NOT originate from structlog.
+
+    Lets stdlib log records (e.g. third-party libraries) flow through the
+    same processors so they render consistently with structlog events.
+    """
+    return [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        structlog.processors.TimeStamper(fmt="iso", utc=True),
+        structlog.processors.format_exc_info,
+    ]
+
+
 def _resolve_level(level_name: str, logger_name: str) -> int:
     try:
         return _parse_level(level_name)
@@ -346,7 +396,7 @@ def configure_logging(config: LogConfig = _DEFAULT_LOG_CONFIG) -> None:
             structlog.processors.TimeStamper(fmt="iso", utc=True),
             structlog.processors.format_exc_info,
             *config.extra_processors,
-            _resolve_renderer(config),
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
         ],
         wrapper_class=structlog.make_filtering_bound_logger(level),
         logger_factory=structlog.stdlib.LoggerFactory(),

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/src/loom/core/observability/observer/structlog.py

@@ -45,14 +45,18 @@ class StructlogLifecycleObserver:
             return
         match event.kind:
             case EventKind.START:
-                bound.debug(event.kind.value, **event.meta)
+                if event.scope is Scope.PIPELINE:
+                    bound.info(event.kind.value, **event.meta)
+                else:
+                    bound.debug(event.kind.value, **event.meta)
             case EventKind.END:
                 bound.info(
                     event.kind.value,
                     duration_ms=event.duration_ms,
                     status=event.status.value if event.status is not None else None,
-                    **event.meta,
                 )
+                if event.meta:
+                    bound.debug(event.kind.value, **event.meta)
             case EventKind.ERROR:
                 bound.error(
                     event.kind.value,

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/src/loom/core/observability/runtime.py

@@ -209,12 +209,18 @@ class ObservabilityRuntime:
         """
         start = perf_counter()
         meta_dict = dict(meta)
+        # Promote ``id`` from meta to the LifecycleEvent's first-class ``id``
+        # field so observers can correlate START/END/ERROR events without
+        # reaching into the ``meta`` dict.
+        _raw_event_id = meta_dict.pop("id", None)
+        event_id: str | None = str(_raw_event_id) if _raw_event_id is not None else None
         self.emit(
             LifecycleEvent.start(
                 scope=scope,
                 name=name,
                 trace_id=trace_id,
                 correlation_id=correlation_id,
+                id=event_id,
                 meta=meta_dict,
             )
         )
@@ -227,6 +233,7 @@ class ObservabilityRuntime:
                     name=name,
                     trace_id=trace_id,
                     correlation_id=correlation_id,
+                    id=event_id,
                     duration_ms=(perf_counter() - start) * 1000,
                     error=str(exc),
                     meta={**meta_dict, "error_type": type(exc).__name__},
@@ -240,6 +247,7 @@ class ObservabilityRuntime:
                     name=name,
                     trace_id=trace_id,
                     correlation_id=correlation_id,
+                    id=event_id,
                     duration_ms=(perf_counter() - start) * 1000,
                     status=LifecycleStatus.SUCCESS,
                     meta=meta_dict,

{loom_kernel-0.9.2 → loom_kernel-0.10.0}/src/loom/etl/backends/polars/_reader.py

@@ -136,7 +136,7 @@ class PolarsSourceReader(SourceReader):
         self._mongo_reader: MongoSourceReader | None = mongo_reader
         self._clickhouse_reader: ClickHouseSourceReader | None = clickhouse_reader
 
-    def read(self, spec: SourceSpec, params_instance: Any) -> pl.LazyFrame:
+    def read(self, spec: SourceSpec, params_instance: Any, /) -> pl.LazyFrame:
         """Read source spec and return lazy frame."""
         if isinstance(spec, TableSourceSpec):
             return self._read_table(spec, params_instance)
@@ -151,6 +151,32 @@ class PolarsSourceReader(SourceReader):
             "TEMP sources are handled by CheckpointStore."
         )
 
+    def read_streaming(self, spec: SourceSpec, params_instance: Any, /) -> pl.LazyFrame:
+        """Route streaming reads to the matching capability-aware sub-reader.
+
+        Only ``ClickHouseSourceSpec`` is supported today; other kinds raise
+        :class:`TypeError`. We refuse to silently fall back to ``read()``
+        because a step that opts into streaming for memory safety must fail
+        loudly rather than risk OOM on a non-streaming path.
+
+        Args:
+            spec: Compiled source specification.
+            params_instance: Concrete params for current run.
+
+        Returns:
+            Lazy frame whose ``collect(engine="streaming")`` is memory-bounded.
+
+        Raises:
+            TypeError: When *spec* is not a ClickHouse spec, or the matching
+                sub-reader was not configured at construction time.
+        """
+        if isinstance(spec, ClickHouseSourceSpec):
+            return self._read_clickhouse_streaming(spec, params_instance)
+        raise TypeError(
+            f"PolarsSourceReader does not support streaming for source kind "
+            f"{spec.kind!r}. Streaming is only available for ClickHouse sources."
+        )
+
     def _read_mongo(self, spec: MongoSourceSpec, params_instance: Any) -> pl.LazyFrame:
         if self._mongo_reader is None:
             raise TypeError(
@@ -160,7 +186,11 @@ class PolarsSourceReader(SourceReader):
         frame: pl.LazyFrame = self._mongo_reader.read(spec, params_instance)
         return frame
 
-    def _read_clickhouse(self, spec: ClickHouseSourceSpec, params_instance: Any) -> pl.LazyFrame:
+    def _read_clickhouse(
+        self,
+        spec: ClickHouseSourceSpec,
+        params_instance: Any,
+    ) -> pl.LazyFrame:
         if self._clickhouse_reader is None:
             raise TypeError(
                 "ClickHouseSourceSpec requires a ClickHouseSourceReader "
@@ -170,6 +200,20 @@ class PolarsSourceReader(SourceReader):
         frame: pl.LazyFrame = self._clickhouse_reader.read(spec, params_instance)
         return frame
 
+    def _read_clickhouse_streaming(
+        self,
+        spec: ClickHouseSourceSpec,
+        params_instance: Any,
+    ) -> pl.LazyFrame:
+        if self._clickhouse_reader is None:
+            raise TypeError(
+                "ClickHouseSourceSpec requires a ClickHouseSourceReader "
+                "injected at construction time. "
+                "Set clickhouse_reader= when constructing PolarsSourceReader."
+            )
+        frame: pl.LazyFrame = self._clickhouse_reader.read_streaming(spec, params_instance)
+        return frame
+
     def execute_sql(self, frames: dict[str, Any], query: str) -> pl.LazyFrame:
         """Execute SQL query against backend frames."""
         return execute_sql(frames, query)