loom-kernel 0.2.1__tar.gz → 0.4.0__tar.gz

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/.github/workflows/ci-pr.yml

@@ -188,7 +188,7 @@ jobs:
             --skip-existing dist/*
 
       - name: Enforce changelog generation
-        if: ${{ always() }}
+        if: ${{ always() && !startsWith(github.head_ref, 'docs/release-') }}
         shell: bash
         run: |
           set -euo pipefail

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/.github/workflows/release.yml

@@ -14,6 +14,7 @@ on:
 permissions:
   contents: write
   id-token: write
+  pull-requests: write
 
 concurrency:
   group: release-${{ github.ref }}
@@ -30,6 +31,7 @@ jobs:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -85,6 +87,43 @@ jobs:
           sed -i "0,/^version = \".*\"/s//version = \"${VERSION}\"/" pyproject.toml
           sed -i "/^\[tool.commitizen\]/,/^\[/ s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
 
+      - name: Open version bump PR
+        if: ${{ steps.version.outputs.deploy == 'true' }}
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          BRANCH="docs/release-v${VERSION}"
+          git checkout -B "${BRANCH}"
+
+          if [ -f CHANGELOG_RELEASE.md ]; then
+            HEADER=$(head -1 CHANGELOG_RELEASE.md)
+            if ! grep -qF "${HEADER}" CHANGELOG.md; then
+              cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
+              mv CHANGELOG_MERGED.md CHANGELOG.md
+            fi
+          fi
+
+          git add pyproject.toml CHANGELOG.md
+          git diff --staged --quiet && exit 0
+          git commit -m "chore(release): bump version to ${VERSION}"
+          git push origin "${BRANCH}" --force
+
+          if ! gh pr list --head "${BRANCH}" --state open --json number --jq '.[0].number' | grep -qE '^[0-9]+$'; then
+            gh pr create \
+              --title "chore(release): bump version to ${VERSION}" \
+              --body "Automated version bump and changelog update after release v${VERSION}." \
+              --base master \
+              --head "${BRANCH}"
+          fi
+
+          gh pr merge "${BRANCH}" --squash --auto --delete-branch
+
       - name: Build package
         if: ${{ steps.version.outputs.deploy == 'true' }}
         shell: bash

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/.gitignore

@@ -19,6 +19,7 @@ htmlcov/
 docs/__pycache__/
 .claude/
 .claude.md
+CLAUDE.md
 AGENTS.md
 
 # Internal discussion docs (not part of distributable source)

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/.readthedocs.yaml

@@ -13,3 +13,9 @@ python:
     - requirements: docs/requirements.txt
     - method: pip
       path: .
+      extra_requirements:
+        - rest
+        - sqlalchemy
+        - cache
+        - config
+        - prometheus

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/CHANGELOG.md

@@ -1,43 +1,109 @@
-# 🚀 Release 0.2.1 ([#12](https://github.com/the-reacher-data/loom-py/pull/12)) ([`87f7d1f`](https://github.com/the-reacher-data/loom-py/commit/87f7d1f1eb1ccde71d0aca1c5584b83317e30707))
+# 🚀 Release 0.4.0 ([#18](https://github.com/the-reacher-data/loom-py/pull/18)) ([`09e5aa3`](https://github.com/the-reacher-data/loom-py/commit/09e5aa340f88e0488daca2bf10320aad7aebbe1d))
 
 
 ## ✨ Features
-### logger
-- **logger:** support per-logger levels from config
+### etl
+- **etl:** add IntoHistory builder and SCD Type 2 domain contracts
 
-### repository
-- **repository:** generalize main repo registration for loom structs
 
 
-## 🐛 Fixes
-### rest
-- **rest:** serialize pagination envelopes in camel case
-- **rest:** support loom structs in autocrud tests
 
-### prometheus
-- **prometheus:** expose metrics at exact path
 
-### review
-- **review:** apply quick-fix pass from code review
+
+
+
+
+
+
+# 🚀 Release 0.3.0 ([#14](https://github.com/the-reacher-data/loom-py/pull/14)) ([`ef414c5`](https://github.com/the-reacher-data/loom-py/commit/ef414c5bfd303296af450840318dfbe9d301e5d1))
+
+
+## ✨ Features
+### config
+- **config:** add cloud URI support and pluggable resolver extension point<br>
+  > Add fsspec as a hard dependency of loom[config]
+  > load_config() now accepts s3://, gs://, abfss://, r2:// URIs via fsspec
+  > Add ConfigResolver protocol for pluggable ${prefix:key} resolution at
+  > parse time (enables SSM, Key Vault, etc. without baking secrets into images)
+  > Resolver registration is idempotent; resolvers are evaluated at job startup
+  > so secret rotation takes effect on the next run
+  > Migrate loom.etl.runner.config_loader to use core load_config, removing
+  > the parallel OmegaConf implementation
+  > ETL _load_yaml inherits cloud URI and resolver support transparently
+
+
+### etl
+- **etl:** add FileLocator with explicit alias API for file routes<br>
+  > Introduces `FileLocator` protocol and `MappingFileLocator` so that
+  > `FromFile.alias("name")` / `IntoFile.alias("name")` specs resolve at
+  > runtime through `storage.files` config rather than hard-coded URIs.
+  > `FileLocation` / `FileLocator` / `MappingFileLocator` in `storage/_file_locator.py`
+  > `StorageConfig.to_file_locator()` returns `MappingFileLocator | None`
+  > (None when `files` is empty — no conditional needed at call sites)
+  > `FromFile.alias()` / `IntoFile.alias()` classmethods set `is_alias=True`
+  > on the emitted spec
+  > `is_alias: bool` added to `FileSourceSpec` and `FileSpec`
+  > Polars and Spark backends resolve aliases via injected `file_locator`
+  > Both providers wired: `file_locator = config.to_file_locator()`
+  > Full test coverage across io, storage, and backend layers
+
+
+
+## 🐛 Fixes
+### observability
+- **observability:** honor missing table policy for record store writers
 
 
 ## 📖 Documentation
-### repository
-- **repository:** align custom repo examples
+### etl
+- **etl:** keep only user guide and drop refactor docs
+- **etl:** expand ETL documentation and update directory table<br>
+  > Add dummy-loom-etl companion repo link in README and etl guide
+  > Expand README subpaths table with loom.etl and loom.core.config entries
+  > Add FileLocator/alias API, cloud config URI, and ConfigResolver sections to etl guide
+  > Add loom.etl.backends (polars + spark) to etl.rst API reference
+  > Add loom.core.config to core.rst API reference
+
 
 
 
 ## ♻️ Refactor
-### repository
-- **repository:** clarify sqlalchemy registration builder
-- **repository:** inject default repository builder
-- **repository:** simplify registration module
+### stepsql
+- **stepsql:** delegate SQL execution to backend readers
+
+
+
+
 
+# 🚀 Release 0.2.1 ([#12](https://github.com/the-reacher-data/loom-py/pull/12)) ([`87f7d1f`](https://github.com/the-reacher-data/loom-py/commit/87f7d1f1eb1ccde71d0aca1c5584b83317e30707))
 
+## ✨ Features
 
+### logger
+- **logger:** support per-logger levels from config<br>
+  > `LoomConfig` now accepts a `loggers` mapping to override the log level per named logger. Resolves `structlog` / stdlib incompatibility when mixing loom-managed and third-party loggers.
+
+### repository
+- **repository:** generalize main repo registration for loom structs<br>
+  > `repository_for` is now importable from `loom.core.repository` (top-level). The SQLAlchemy-specific import path still works but is no longer the canonical one.
 
+## 🐛 Bug Fixes
+
+### rest
+- **rest:** serialize pagination envelopes in camel case<br>
+  > `PageResult` and list-envelope responses were serialized in snake_case. All envelope fields now follow the camelCase contract of the HTTP layer.
+- **rest:** support loom structs in autocrud tests<br>
+  > Auto-CRUD route generation was not exercising the `msgspec.Struct` code path in integration tests.
+
+### prometheus
+- **prometheus:** expose metrics at exact path<br>
+  > Metrics endpoint was registered with a trailing-slash variant that did not match the documented `/metrics` path.
 
+### docs
+- **docs:** fix RTD build failure, logo and docs examples (#10, #11)<br>
+  > Mock `starlette`, `celery`, `kombu`, `redis` in `autodoc_mock_imports`. Logo resized to natural proportions with dark-mode safe background. Status badges added to index. Rule/Compute examples updated to named predicates.
 
+---
 
 # 🚀 Release 0.2.0 ([#9](https://github.com/the-reacher-data/loom-py/pull/9)) ([`2f669ab`](https://github.com/the-reacher-data/loom-py/commit/2f669ab205c7255eb6494e4cdb8ab8092817af62))
 

loom_kernel-0.4.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,15 @@
+# 🚀 Release 0.4.0 ([#18](https://github.com/the-reacher-data/loom-py/pull/18)) ([`09e5aa3`](https://github.com/the-reacher-data/loom-py/commit/09e5aa340f88e0488daca2bf10320aad7aebbe1d))
+
+
+## ✨ Features
+### etl
+- **etl:** add IntoHistory builder and SCD Type 2 domain contracts
+
+
+
+
+
+
+
+
+

loom_kernel-0.2.1/README.md → loom_kernel-0.4.0/PKG-INFO

@@ -1,3 +1,61 @@
+Metadata-Version: 2.4
+Name: loom-kernel
+Version: 0.4.0
+Summary: Loom Python project
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: aiocache<1.0,>=0.12
+Requires-Dist: msgspec<1.0,>=0.18
+Requires-Dist: omegaconf<3.0,>=2.3
+Requires-Dist: prometheus-client>=0.20
+Requires-Dist: structlog<26.0,>=24.4
+Provides-Extra: cache
+Requires-Dist: aiocache<1.0,>=0.12; extra == 'cache'
+Requires-Dist: omegaconf<3.0,>=2.3; extra == 'cache'
+Provides-Extra: celery
+Requires-Dist: celery<6.0,>=5.3; extra == 'celery'
+Requires-Dist: kombu<6.0,>=5.3; extra == 'celery'
+Requires-Dist: redis<6.0,>=5.0; extra == 'celery'
+Provides-Extra: config
+Requires-Dist: fsspec>=2024.2.0; extra == 'config'
+Requires-Dist: omegaconf<3.0,>=2.3; extra == 'config'
+Provides-Extra: etl-otel
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-otel'
+Provides-Extra: etl-polars
+Requires-Dist: deltalake<2.0,>=1.5; extra == 'etl-polars'
+Requires-Dist: fsspec>=2024.2.0; extra == 'etl-polars'
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: polars<2.0,>=1.0; extra == 'etl-polars'
+Provides-Extra: etl-spark
+Requires-Dist: delta-spark<4.0,>=3.2; extra == 'etl-spark'
+Requires-Dist: findspark<3.0,>=2.0; extra == 'etl-spark'
+Requires-Dist: fsspec>=2024.2.0; extra == 'etl-spark'
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: pyspark<4.0,>=3.5; extra == 'etl-spark'
+Provides-Extra: prometheus
+Requires-Dist: prometheus-client>=0.20; extra == 'prometheus'
+Provides-Extra: pyspark
+Requires-Dist: pyspark<4.0,>=3.5; extra == 'pyspark'
+Provides-Extra: rest
+Requires-Dist: fastapi<1.0,>=0.115; extra == 'rest'
+Requires-Dist: httptools<1.0,>=0.6; extra == 'rest'
+Requires-Dist: prometheus-client>=0.20; extra == 'rest'
+Requires-Dist: pydantic<3.0,>=2.0; extra == 'rest'
+Requires-Dist: uvicorn<1.0,>=0.30; extra == 'rest'
+Requires-Dist: uvloop<1.0,>=0.21; (sys_platform != 'win32') and extra == 'rest'
+Provides-Extra: sqlalchemy
+Requires-Dist: sqlalchemy<3.0,>=2.0; extra == 'sqlalchemy'
+Description-Content-Type: text/markdown
+
 <p align="center">
   <img src="docs/_static/logo-transparent.png" alt="loom-kernel" width="160" style="background:#ffffff;border-radius:6px;padding:8px 20px;" />
 </p>
@@ -20,7 +78,8 @@ Framework-agnostic Python toolkit to build backend applications with:
 - repositories decoupled from infrastructure
 - REST/FastAPI adapters with OpenAPI generation
 - background jobs and Celery workers, first-class
-- testing utilities for business workflows
+- **declarative ETL** — compile-time-validated pipelines for Polars and Spark
+- testing utilities for business workflows and ETL steps
 
 ## Purpose
 
@@ -35,7 +94,8 @@ infrastructure (DB, cache, transport) without breaking business logic.
 
 - Usage guides and architecture docs are available in the `docs/` site.
 - API reference is autogenerated from public docstrings.
-- End-to-end demo application: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end REST demo: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end ETL demo: [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl) — full Polars and Spark pipeline examples.
 
 ## Main subpaths
 
@@ -47,10 +107,23 @@ infrastructure (DB, cache, transport) without breaking business logic.
 | `src/loom/core/repository/sqlalchemy` | Concrete async SQLAlchemy repository implementation. |
 | `src/loom/core/model` | Base model, fields, relations, and entity introspection. |
 | `src/loom/core/cache` | Decorators and cached repository with dependency invalidation. |
+| `src/loom/core/config` | YAML config loader with cloud URI support and pluggable resolvers. |
 | `src/loom/rest` | Framework-agnostic REST model and route compiler. |
 | `src/loom/rest/fastapi` | Direct FastAPI integration (auto wiring and runtime router). |
 | `src/loom/prometheus` | Middleware and adapter for runtime metrics. |
 | `src/loom/testing` | Harnesses for unit/integration tests and golden tests. |
+| `src/loom/etl` | Declarative ETL subsystem — pipelines, Polars/Spark backends, observability. |
+| `src/loom/etl/pipeline` | `ETLStep`, `ETLProcess`, `ETLPipeline`, `StepSQL`, and `ETLParams`. |
+| `src/loom/etl/declarative` | `FromTable`, `FromFile`, `IntoTable`, `IntoFile`, predicate DSL, and `params` proxy. |
+| `src/loom/etl/schema` | Backend-agnostic schema model (`LoomDtype`, `ColumnSchema`, `TableRef`). |
+| `src/loom/etl/storage` | Storage config, table/file locators, and route resolution. |
+| `src/loom/etl/compiler` | Compile-time validation and execution plan builder. |
+| `src/loom/etl/runner` | `ETLRunner` entry point and YAML config loader. |
+| `src/loom/etl/checkpoint` | Step-level checkpoint store for incremental re-runs. |
+| `src/loom/etl/observability` | Run/step observers, OTEL sink, structlog sink, and execution records. |
+| `src/loom/etl/backends/polars` | Polars + Delta Lake reader, writer, and schema aligner. |
+| `src/loom/etl/backends/spark` | Spark + Delta reader, writer, and schema aligner. |
+| `src/loom/etl/testing` | `PolarsStepRunner`, `SparkStepRunner`, `ETLScenario`, and test stubs. |
 
 ## Quick start
 
@@ -380,6 +453,66 @@ For deeper references, review the integration examples under
 For a runnable full-stack sample with all patterns combined, check the companion repository
 [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
 
+## ETL quick start
+
+Install a backend:
+
+```bash
+pip install "loom-kernel[etl-polars]"
+# or
+pip install "loom-kernel[etl-spark]"
+```
+
+Declare a pipeline — sources, targets, and transformation logic are explicit and compile-time validated:
+
+```python
+from datetime import date
+import polars as pl
+from loom.etl import ETLParams, ETLStep, ETLProcess, ETLPipeline, ETLRunner, FromTable, IntoTable
+
+class DailyParams(ETLParams):
+    run_date: date
+
+class CleanOrders(ETLStep[DailyParams]):
+    orders = FromTable("raw.orders").columns("id", "amount", "run_date")
+    target = IntoTable("staging.orders").replace()
+
+    def execute(self, params: DailyParams, *, orders: pl.LazyFrame) -> pl.LazyFrame:
+        return orders.filter(pl.col("amount") > 0)
+
+class DailyProcess(ETLProcess[DailyParams]):
+    steps = [CleanOrders]
+
+class DailyPipeline(ETLPipeline[DailyParams]):
+    processes = [DailyProcess]
+
+runner = ETLRunner.from_yaml("config/etl.yaml")
+runner.run(DailyPipeline, DailyParams(run_date=date.today()))
+```
+
+Write modes — `replace`, `append`, `replace_partition`, `replace_partitions`, `replace_where`, `upsert` — are declared on the target, validated at compile time. Partition predicates use the `params` proxy so no values are hard-coded:
+
+```python
+from loom.etl import params
+
+target = IntoTable("staging.orders").replace_partition(
+    year=params.run_date.year,
+    month=params.run_date.month,
+)
+```
+
+File aliases decouple paths from pipelines — declare once in YAML, reference by name:
+
+```python
+events = FromFile.alias("events_raw", format=Format.CSV)
+target = IntoFile.alias("exports_daily", format=Format.PARQUET)
+```
+
+For a full Polars + Spark + Delta Lake example see [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl).
+For the complete API reference and write-mode guide see the [ETL docs](https://loom-py.readthedocs.io/en/latest/guides/etl.html).
+
+---
+
 ## Performance
 
 `loom-kernel` adds zero measurable overhead at the concurrency levels typical of

loom_kernel-0.2.1/PKG-INFO → loom_kernel-0.4.0/README.md

@@ -1,38 +1,3 @@
-Metadata-Version: 2.4
-Name: loom-kernel
-Version: 0.2.1
-Summary: Loom Python project
-License-File: LICENSE
-Requires-Python: >=3.11
-Requires-Dist: aiocache<1.0,>=0.12
-Requires-Dist: msgspec<1.0,>=0.18
-Requires-Dist: omegaconf<3.0,>=2.3
-Requires-Dist: prometheus-client>=0.20
-Requires-Dist: structlog<26.0,>=24.4
-Provides-Extra: cache
-Requires-Dist: aiocache<1.0,>=0.12; extra == 'cache'
-Requires-Dist: omegaconf<3.0,>=2.3; extra == 'cache'
-Provides-Extra: celery
-Requires-Dist: celery<6.0,>=5.3; extra == 'celery'
-Requires-Dist: kombu<6.0,>=5.3; extra == 'celery'
-Requires-Dist: redis<6.0,>=5.0; extra == 'celery'
-Provides-Extra: config
-Requires-Dist: omegaconf<3.0,>=2.3; extra == 'config'
-Provides-Extra: prometheus
-Requires-Dist: prometheus-client>=0.20; extra == 'prometheus'
-Provides-Extra: pyspark
-Requires-Dist: pyspark<4.0,>=3.5; extra == 'pyspark'
-Provides-Extra: rest
-Requires-Dist: fastapi<1.0,>=0.115; extra == 'rest'
-Requires-Dist: httptools<1.0,>=0.6; extra == 'rest'
-Requires-Dist: prometheus-client>=0.20; extra == 'rest'
-Requires-Dist: pydantic<3.0,>=2.0; extra == 'rest'
-Requires-Dist: uvicorn<1.0,>=0.30; extra == 'rest'
-Requires-Dist: uvloop<1.0,>=0.21; (sys_platform != 'win32') and extra == 'rest'
-Provides-Extra: sqlalchemy
-Requires-Dist: sqlalchemy<3.0,>=2.0; extra == 'sqlalchemy'
-Description-Content-Type: text/markdown
-
 <p align="center">
   <img src="docs/_static/logo-transparent.png" alt="loom-kernel" width="160" style="background:#ffffff;border-radius:6px;padding:8px 20px;" />
 </p>
@@ -55,7 +20,8 @@ Framework-agnostic Python toolkit to build backend applications with:
 - repositories decoupled from infrastructure
 - REST/FastAPI adapters with OpenAPI generation
 - background jobs and Celery workers, first-class
-- testing utilities for business workflows
+- **declarative ETL** — compile-time-validated pipelines for Polars and Spark
+- testing utilities for business workflows and ETL steps
 
 ## Purpose
 
@@ -70,7 +36,8 @@ infrastructure (DB, cache, transport) without breaking business logic.
 
 - Usage guides and architecture docs are available in the `docs/` site.
 - API reference is autogenerated from public docstrings.
-- End-to-end demo application: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end REST demo: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end ETL demo: [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl) — full Polars and Spark pipeline examples.
 
 ## Main subpaths
 
@@ -82,10 +49,23 @@ infrastructure (DB, cache, transport) without breaking business logic.
 | `src/loom/core/repository/sqlalchemy` | Concrete async SQLAlchemy repository implementation. |
 | `src/loom/core/model` | Base model, fields, relations, and entity introspection. |
 | `src/loom/core/cache` | Decorators and cached repository with dependency invalidation. |
+| `src/loom/core/config` | YAML config loader with cloud URI support and pluggable resolvers. |
 | `src/loom/rest` | Framework-agnostic REST model and route compiler. |
 | `src/loom/rest/fastapi` | Direct FastAPI integration (auto wiring and runtime router). |
 | `src/loom/prometheus` | Middleware and adapter for runtime metrics. |
 | `src/loom/testing` | Harnesses for unit/integration tests and golden tests. |
+| `src/loom/etl` | Declarative ETL subsystem — pipelines, Polars/Spark backends, observability. |
+| `src/loom/etl/pipeline` | `ETLStep`, `ETLProcess`, `ETLPipeline`, `StepSQL`, and `ETLParams`. |
+| `src/loom/etl/declarative` | `FromTable`, `FromFile`, `IntoTable`, `IntoFile`, predicate DSL, and `params` proxy. |
+| `src/loom/etl/schema` | Backend-agnostic schema model (`LoomDtype`, `ColumnSchema`, `TableRef`). |
+| `src/loom/etl/storage` | Storage config, table/file locators, and route resolution. |
+| `src/loom/etl/compiler` | Compile-time validation and execution plan builder. |
+| `src/loom/etl/runner` | `ETLRunner` entry point and YAML config loader. |
+| `src/loom/etl/checkpoint` | Step-level checkpoint store for incremental re-runs. |
+| `src/loom/etl/observability` | Run/step observers, OTEL sink, structlog sink, and execution records. |
+| `src/loom/etl/backends/polars` | Polars + Delta Lake reader, writer, and schema aligner. |
+| `src/loom/etl/backends/spark` | Spark + Delta reader, writer, and schema aligner. |
+| `src/loom/etl/testing` | `PolarsStepRunner`, `SparkStepRunner`, `ETLScenario`, and test stubs. |
 
 ## Quick start
 
@@ -415,6 +395,66 @@ For deeper references, review the integration examples under
 For a runnable full-stack sample with all patterns combined, check the companion repository
 [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
 
+## ETL quick start
+
+Install a backend:
+
+```bash
+pip install "loom-kernel[etl-polars]"
+# or
+pip install "loom-kernel[etl-spark]"
+```
+
+Declare a pipeline — sources, targets, and transformation logic are explicit and compile-time validated:
+
+```python
+from datetime import date
+import polars as pl
+from loom.etl import ETLParams, ETLStep, ETLProcess, ETLPipeline, ETLRunner, FromTable, IntoTable
+
+class DailyParams(ETLParams):
+    run_date: date
+
+class CleanOrders(ETLStep[DailyParams]):
+    orders = FromTable("raw.orders").columns("id", "amount", "run_date")
+    target = IntoTable("staging.orders").replace()
+
+    def execute(self, params: DailyParams, *, orders: pl.LazyFrame) -> pl.LazyFrame:
+        return orders.filter(pl.col("amount") > 0)
+
+class DailyProcess(ETLProcess[DailyParams]):
+    steps = [CleanOrders]
+
+class DailyPipeline(ETLPipeline[DailyParams]):
+    processes = [DailyProcess]
+
+runner = ETLRunner.from_yaml("config/etl.yaml")
+runner.run(DailyPipeline, DailyParams(run_date=date.today()))
+```
+
+Write modes — `replace`, `append`, `replace_partition`, `replace_partitions`, `replace_where`, `upsert` — are declared on the target, validated at compile time. Partition predicates use the `params` proxy so no values are hard-coded:
+
+```python
+from loom.etl import params
+
+target = IntoTable("staging.orders").replace_partition(
+    year=params.run_date.year,
+    month=params.run_date.month,
+)
+```
+
+File aliases decouple paths from pipelines — declare once in YAML, reference by name:
+
+```python
+events = FromFile.alias("events_raw", format=Format.CSV)
+target = IntoFile.alias("exports_daily", format=Format.PARQUET)
+```
+
+For a full Polars + Spark + Delta Lake example see [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl).
+For the complete API reference and write-mode guide see the [ETL docs](https://loom-py.readthedocs.io/en/latest/guides/etl.html).
+
+---
+
 ## Performance
 
 `loom-kernel` adds zero measurable overhead at the concurrency levels typical of

{loom_kernel-0.2.1 → loom_kernel-0.4.0}/docs/conf.py

@@ -6,6 +6,7 @@ import os
 import sys
 from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 ROOT = Path(__file__).resolve().parents[1]
 SRC = ROOT / "src"
@@ -28,7 +29,7 @@ extensions = [
 ]
 
 templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "guides", "architecture", "examples-repo"]
 
 html_theme = "furo"
 html_static_path = ["_static"]
@@ -52,7 +53,14 @@ napoleon_google_docstring = True
 napoleon_numpy_docstring = False
 
 
-def _skip_duplicate_reexports(app, what, name, obj, skip, options):
+def _skip_duplicate_reexports(
+    app: Any,
+    what: str,
+    name: str,
+    obj: Any,
+    skip: bool,
+    options: Any,
+) -> bool:
     """Skip known re-exported symbols that duplicate canonical API objects."""
     del what, obj, options
     current_module = app.env.temp_data.get("autodoc:module")
@@ -81,9 +89,13 @@ intersphinx_mapping = {
 autodoc_mock_imports = [
     "aiocache",
     "celery",
+    "deltalake",
     "fastapi",
+    "fsspec",
     "kombu",
     "omegaconf",
+    "polars",
+    "pyarrow",
     "prometheus_client",
     "pydantic",
     "pyspark",
@@ -94,6 +106,6 @@ autodoc_mock_imports = [
 ]
 
 
-def setup(app):
+def setup(app: Any) -> None:
     """Register Sphinx hooks."""
     app.connect("autodoc-skip-member", _skip_duplicate_reexports)

loom_kernel-0.4.0/docs/etl/examples.md

@@ -0,0 +1,29 @@
+# ETL Examples
+
+The companion repository [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl)
+contains runnable end-to-end examples of `loom.etl` with both Polars and Spark
+backends.
+
+## What it covers
+
+- Polars and Spark pipeline declarations
+- Delta Lake read/write modes (`replace`, `upsert`, `replace_partitions`)
+- Cloud storage configuration (S3, GCS, Azure) via `fsspec`
+- File aliases with `FromFile.alias` / `IntoFile.alias`
+- Observability and run sinks
+- Integration tests with `PolarsStepRunner` and `SparkTestSession`
+
+## Quick start
+
+```bash
+git clone https://github.com/the-reacher-data/dummy-loom-etl
+cd dummy-loom-etl
+make test      # run Polars and Spark test suites
+make run       # run the sample pipeline locally
+```
+
+## Next steps
+
+- [ETL pipelines guide](pipelines.md)
+- [ETL testing guide](testing.md)
+- [API reference](../reference/api/etl)