loom-kernel 0.2.0__tar.gz → 0.3.0__tar.gz

{loom_kernel-0.2.0 → loom_kernel-0.3.0}/.github/workflows/ci-pr.yml

@@ -188,7 +188,7 @@ jobs:
             --skip-existing dist/*
 
       - name: Enforce changelog generation
-        if: ${{ always() }}
+        if: ${{ always() && !startsWith(github.head_ref, 'docs/release-') }}
         shell: bash
         run: |
           set -euo pipefail

{loom_kernel-0.2.0 → loom_kernel-0.3.0}/.github/workflows/release.yml

@@ -14,6 +14,7 @@ on:
 permissions:
   contents: write
   id-token: write
+  pull-requests: write
 
 concurrency:
   group: release-${{ github.ref }}
@@ -30,6 +31,7 @@ jobs:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -85,6 +87,38 @@ jobs:
           sed -i "0,/^version = \".*\"/s//version = \"${VERSION}\"/" pyproject.toml
           sed -i "/^\[tool.commitizen\]/,/^\[/ s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
 
+      - name: Open version bump PR
+        if: ${{ steps.version.outputs.deploy == 'true' }}
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          BRANCH="docs/release-v${VERSION}"
+          git checkout -b "${BRANCH}"
+
+          if [ -f CHANGELOG_RELEASE.md ]; then
+            cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
+            mv CHANGELOG_MERGED.md CHANGELOG.md
+          fi
+
+          git add pyproject.toml CHANGELOG.md
+          git diff --staged --quiet && exit 0
+          git commit -m "chore(release): bump version to ${VERSION}"
+          git push origin "${BRANCH}"
+
+          gh pr create \
+            --title "chore(release): bump version to ${VERSION}" \
+            --body "Automated version bump and changelog update after release v${VERSION}." \
+            --base master \
+            --head "${BRANCH}"
+
+          gh pr merge "${BRANCH}" --squash --auto --delete-branch
+
       - name: Build package
         if: ${{ steps.version.outputs.deploy == 'true' }}
         shell: bash

loom_kernel-0.3.0/CHANGELOG.md

@@ -0,0 +1,216 @@
+# 🚀 Release 0.3.0 ([#14](https://github.com/the-reacher-data/loom-py/pull/14)) ([`ef414c5`](https://github.com/the-reacher-data/loom-py/commit/ef414c5bfd303296af450840318dfbe9d301e5d1))
+
+
+## ✨ Features
+### config
+- **config:** add cloud URI support and pluggable resolver extension point<br>
+  > Add fsspec as a hard dependency of loom[config]
+  > load_config() now accepts s3://, gs://, abfss://, r2:// URIs via fsspec
+  > Add ConfigResolver protocol for pluggable ${prefix:key} resolution at
+  > parse time (enables SSM, Key Vault, etc. without baking secrets into images)
+  > Resolver registration is idempotent; resolvers are evaluated at job startup
+  > so secret rotation takes effect on the next run
+  > Migrate loom.etl.runner.config_loader to use core load_config, removing
+  > the parallel OmegaConf implementation
+  > ETL _load_yaml inherits cloud URI and resolver support transparently
+
+
+### etl
+- **etl:** add FileLocator with explicit alias API for file routes<br>
+  > Introduces `FileLocator` protocol and `MappingFileLocator` so that
+  > `FromFile.alias("name")` / `IntoFile.alias("name")` specs resolve at
+  > runtime through `storage.files` config rather than hard-coded URIs.
+  > `FileLocation` / `FileLocator` / `MappingFileLocator` in `storage/_file_locator.py`
+  > `StorageConfig.to_file_locator()` returns `MappingFileLocator | None`
+  > (None when `files` is empty — no conditional needed at call sites)
+  > `FromFile.alias()` / `IntoFile.alias()` classmethods set `is_alias=True`
+  > on the emitted spec
+  > `is_alias: bool` added to `FileSourceSpec` and `FileSpec`
+  > Polars and Spark backends resolve aliases via injected `file_locator`
+  > Both providers wired: `file_locator = config.to_file_locator()`
+  > Full test coverage across io, storage, and backend layers
+
+
+
+## 🐛 Fixes
+### observability
+- **observability:** honor missing table policy for record store writers
+
+
+## 📖 Documentation
+### etl
+- **etl:** keep only user guide and drop refactor docs
+- **etl:** expand ETL documentation and update directory table<br>
+  > Add dummy-loom-etl companion repo link in README and etl guide
+  > Expand README subpaths table with loom.etl and loom.core.config entries
+  > Add FileLocator/alias API, cloud config URI, and ConfigResolver sections to etl guide
+  > Add loom.etl.backends (polars + spark) to etl.rst API reference
+  > Add loom.core.config to core.rst API reference
+
+
+
+
+## ♻️ Refactor
+### stepsql
+- **stepsql:** delegate SQL execution to backend readers
+
+
+
+
+
+# 🚀 Release 0.3.0 ([#14](https://github.com/the-reacher-data/loom-py/pull/14)) ([`ef414c5`](https://github.com/the-reacher-data/loom-py/commit/ef414c5bfd303296af450840318dfbe9d301e5d1))
+
+
+## ✨ Features
+### config
+- **config:** add cloud URI support and pluggable resolver extension point<br>
+  > Add fsspec as a hard dependency of loom[config]
+  > load_config() now accepts s3://, gs://, abfss://, r2:// URIs via fsspec
+  > Add ConfigResolver protocol for pluggable ${prefix:key} resolution at
+  > parse time (enables SSM, Key Vault, etc. without baking secrets into images)
+  > Resolver registration is idempotent; resolvers are evaluated at job startup
+  > so secret rotation takes effect on the next run
+  > Migrate loom.etl.runner.config_loader to use core load_config, removing
+  > the parallel OmegaConf implementation
+  > ETL _load_yaml inherits cloud URI and resolver support transparently
+
+
+### etl
+- **etl:** add FileLocator with explicit alias API for file routes<br>
+  > Introduces `FileLocator` protocol and `MappingFileLocator` so that
+  > `FromFile.alias("name")` / `IntoFile.alias("name")` specs resolve at
+  > runtime through `storage.files` config rather than hard-coded URIs.
+  > `FileLocation` / `FileLocator` / `MappingFileLocator` in `storage/_file_locator.py`
+  > `StorageConfig.to_file_locator()` returns `MappingFileLocator | None`
+  > (None when `files` is empty — no conditional needed at call sites)
+  > `FromFile.alias()` / `IntoFile.alias()` classmethods set `is_alias=True`
+  > on the emitted spec
+  > `is_alias: bool` added to `FileSourceSpec` and `FileSpec`
+  > Polars and Spark backends resolve aliases via injected `file_locator`
+  > Both providers wired: `file_locator = config.to_file_locator()`
+  > Full test coverage across io, storage, and backend layers
+
+
+
+## 🐛 Fixes
+### observability
+- **observability:** honor missing table policy for record store writers
+
+
+## 📖 Documentation
+### etl
+- **etl:** keep only user guide and drop refactor docs
+- **etl:** expand ETL documentation and update directory table<br>
+  > Add dummy-loom-etl companion repo link in README and etl guide
+  > Expand README subpaths table with loom.etl and loom.core.config entries
+  > Add FileLocator/alias API, cloud config URI, and ConfigResolver sections to etl guide
+  > Add loom.etl.backends (polars + spark) to etl.rst API reference
+  > Add loom.core.config to core.rst API reference
+
+
+
+
+## ♻️ Refactor
+### stepsql
+- **stepsql:** delegate SQL execution to backend readers
+
+
+
+
+
+
+
+# 🚀 Release 0.2.1 ([#12](https://github.com/the-reacher-data/loom-py/pull/12)) ([`87f7d1f`](https://github.com/the-reacher-data/loom-py/commit/87f7d1f1eb1ccde71d0aca1c5584b83317e30707))
+
+## ✨ Features
+
+### logger
+- **logger:** support per-logger levels from config<br>
+  > `LoomConfig` now accepts a `loggers` mapping to override the log level per named logger. Resolves `structlog` / stdlib incompatibility when mixing loom-managed and third-party loggers.
+
+### repository
+- **repository:** generalize main repo registration for loom structs<br>
+  > `repository_for` is now importable from `loom.core.repository` (top-level). The SQLAlchemy-specific import path still works but is no longer the canonical one.
+
+## 🐛 Bug Fixes
+
+### rest
+- **rest:** serialize pagination envelopes in camel case<br>
+  > `PageResult` and list-envelope responses were serialized in snake_case. All envelope fields now follow the camelCase contract of the HTTP layer.
+- **rest:** support loom structs in autocrud tests<br>
+  > Auto-CRUD route generation was not exercising the `msgspec.Struct` code path in integration tests.
+
+### prometheus
+- **prometheus:** expose metrics at exact path<br>
+  > Metrics endpoint was registered with a trailing-slash variant that did not match the documented `/metrics` path.
+
+### docs
+- **docs:** fix RTD build failure, logo and docs examples (#10, #11)<br>
+  > Mock `starlette`, `celery`, `kombu`, `redis` in `autodoc_mock_imports`. Logo resized to natural proportions with dark-mode safe background. Status badges added to index. Rule/Compute examples updated to named predicates.
+
+---
+
+# 🚀 Release 0.2.0 ([#9](https://github.com/the-reacher-data/loom-py/pull/9)) ([`2f669ab`](https://github.com/the-reacher-data/loom-py/commit/2f669ab205c7255eb6494e4cdb8ab8092817af62))
+
+## ✨ Features
+
+### cache
+- **cache:** aiocache gateway with auto-inferred invalidation specs<br>
+  > CachedRepository wraps any repository with read-through/write-through caching. ONE_TO_MANY depends_on specs are auto-generated from field annotations — no explicit declaration needed. Explicit depends_on always wins.
+
+### celery
+- **celery:** production-ready Celery integration layer<br>
+  > CeleryJobService, persistent worker event loop, trace propagation, eager fallback, and task_default_queue routing so callbacks land on the correct consumed queue. bootstrap_worker compiles use cases, repositories, and registers Celery tasks in a single call.
+
+- **celery:** worker job discovery from modules or manifest<br>
+  > bootstrap_worker discovers and registers Job classes automatically from module include paths (mode: modules) or from a typed WorkerManifest (mode: manifest). WorkerManifest replaces scattered JOBS/USE_CASES/INTERFACES module attributes with a single typed contract.
+
+- **celery:** interfaces= and use_cases= on bootstrap_worker<br>
+  > Callbacks that call ApplicationInvoker need matching use-case keys compiled in the worker. interfaces= extracts use-case types from RestInterface route declarations (including AutoCRUD-generated ones). use_cases= handles non-AutoCRUD scenarios. Both can be combined with discovery mode.
+
+### core
+- **core:** typed repository abstractions and SQLAlchemy backend<br>
+  > Async repository protocol (RepositoryRead, RepositoryWrite, RepoFor) backed by SQLAlchemy 2.0 async session. Struct-based model system using msgspec.Struct as the single source of truth — models compile to SA mapped classes at startup via compile_all(). count() and UPDATE RETURNING included as first-class operations.
+
+- **core:** use-case DSL with field refs, compute, rules and typed markers<br>
+  > Declarative use-case definition via Input, Load, LoadById, Exists, Compute and Rule markers. Signature inspection runs once at compile time; RuntimeExecutor drives execution from an immutable ExecutionPlan. No per-request reflection.
+
+- **core:** ApplicationInvoker and named use-case registry<br>
+  > Use cases and job callbacks invoke other use cases by type through ApplicationInvoker without direct coupling. A named registry maps use-case keys to compiled instances at bootstrap, providing a stable cross-invocation contract.
+
+- **core:** compiled model artifact and cache entity keys<br>
+  > compile_all() produces a typed CompiledCore artifact exposing stable entity keys used by the cache layer for deterministic repository-level invalidation across reads and writes.
+
+- **core:** executor skips UoW for read-only use cases and GET routes<br>
+  > UseCase.read_only=True and all GET routes bypass UoW.begin/commit, removing at minimum one BEGIN+COMMIT round-trip from every read request on PostgreSQL.
+
+### job
+- **job:** async job domain model and orchestration primitives<br>
+  > Job[ResultT] base class with Celery routing ClassVars. JobHandle / JobGroup with dual-mode waiting (Celery + inline). JobCallback lifecycle with on_success/on_failure. Dispatch is transactionally safe — jobs flush on UoW commit and are cleared on rollback.
+
+### observability
+- **observability:** trace_id propagation and Prometheus adapter<br>
+  > trace_id injected into every request context and propagated to job callbacks. MetricsAdapter protocol emits execution events; PrometheusAdapter records latency histograms and error counters with low cardinality labels.
+
+### projection
+- **projection:** compiler-driven memory/SQL routing<br>
+  > Projections are source-agnostic at declaration time. The backend compiler decides at compile_all() whether each projection runs in-memory (relation already loaded in the active profile) or via SQL. Users declare only CountLoader, ExistsLoader, or JoinFieldsLoader — no source= parameter. Internal _Memory* and _Sql* loaders are synthesized at compile time.
+
+### rest
+- **rest:** AutoCRUD and FastAPI adapter<br>
+  > RestInterface.auto=True generates full CRUD routes at class definition time via build_auto_routes(). OpenAPI contracts expose query params, pagination defaults, and decoupled CreateInput/UpdateInput write DTOs. Discovery engine mounts all declared interfaces at bootstrap.
+
+## 📖 Documentation
+
+- Sphinx documentation platform with full public guides<br>
+  > Quickstart, use-case DSL reference, AutoCRUD guide, Celery integration guide (job definition, dispatch, callbacks, YAML reference, bootstrap options, ApplicationInvoker, Docker-compose stack), and dummy-loom examples-repo walkthrough. Deployed to Read the Docs.
+
+## ⚡ Performance
+
+### engine
+- **engine:** UPDATE RETURNING replaces SELECT + flush + refresh<br>
+  > SQLAlchemyUpdateMixin.update() issues a single UPDATE ... RETURNING round-trip. Server-side onupdate expressions are pre-computed at init time and injected into the SET clause automatically.
+
+### repository
+- **repository:** single-query total count for offset pagination<br>
+  > list_with_query with PaginationMode.OFFSET issues a single SELECT COUNT(*) instead of a separate full-table scan, eliminating one round-trip per paginated list operation.

loom_kernel-0.3.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,60 @@
+# 🚀 Release 0.3.0 ([#14](https://github.com/the-reacher-data/loom-py/pull/14)) ([`ef414c5`](https://github.com/the-reacher-data/loom-py/commit/ef414c5bfd303296af450840318dfbe9d301e5d1))
+
+
+## ✨ Features
+### config
+- **config:** add cloud URI support and pluggable resolver extension point<br>
+  > Add fsspec as a hard dependency of loom[config]
+  > load_config() now accepts s3://, gs://, abfss://, r2:// URIs via fsspec
+  > Add ConfigResolver protocol for pluggable ${prefix:key} resolution at
+  > parse time (enables SSM, Key Vault, etc. without baking secrets into images)
+  > Resolver registration is idempotent; resolvers are evaluated at job startup
+  > so secret rotation takes effect on the next run
+  > Migrate loom.etl.runner.config_loader to use core load_config, removing
+  > the parallel OmegaConf implementation
+  > ETL _load_yaml inherits cloud URI and resolver support transparently
+
+
+### etl
+- **etl:** add FileLocator with explicit alias API for file routes<br>
+  > Introduces `FileLocator` protocol and `MappingFileLocator` so that
+  > `FromFile.alias("name")` / `IntoFile.alias("name")` specs resolve at
+  > runtime through `storage.files` config rather than hard-coded URIs.
+  > `FileLocation` / `FileLocator` / `MappingFileLocator` in `storage/_file_locator.py`
+  > `StorageConfig.to_file_locator()` returns `MappingFileLocator | None`
+  > (None when `files` is empty — no conditional needed at call sites)
+  > `FromFile.alias()` / `IntoFile.alias()` classmethods set `is_alias=True`
+  > on the emitted spec
+  > `is_alias: bool` added to `FileSourceSpec` and `FileSpec`
+  > Polars and Spark backends resolve aliases via injected `file_locator`
+  > Both providers wired: `file_locator = config.to_file_locator()`
+  > Full test coverage across io, storage, and backend layers
+
+
+
+## 🐛 Fixes
+### observability
+- **observability:** honor missing table policy for record store writers
+
+
+## 📖 Documentation
+### etl
+- **etl:** keep only user guide and drop refactor docs
+- **etl:** expand ETL documentation and update directory table<br>
+  > Add dummy-loom-etl companion repo link in README and etl guide
+  > Expand README subpaths table with loom.etl and loom.core.config entries
+  > Add FileLocator/alias API, cloud config URI, and ConfigResolver sections to etl guide
+  > Add loom.etl.backends (polars + spark) to etl.rst API reference
+  > Add loom.core.config to core.rst API reference
+
+
+
+
+## ♻️ Refactor
+### stepsql
+- **stepsql:** delegate SQL execution to backend readers
+
+
+
+
+

{loom_kernel-0.2.0 → loom_kernel-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loom-kernel
-Version: 0.2.0
+Version: 0.3.0
 Summary: Loom Python project
 License-File: LICENSE
 Requires-Python: >=3.11
@@ -17,7 +17,30 @@ Requires-Dist: celery<6.0,>=5.3; extra == 'celery'
 Requires-Dist: kombu<6.0,>=5.3; extra == 'celery'
 Requires-Dist: redis<6.0,>=5.0; extra == 'celery'
 Provides-Extra: config
+Requires-Dist: fsspec>=2024.2.0; extra == 'config'
 Requires-Dist: omegaconf<3.0,>=2.3; extra == 'config'
+Provides-Extra: etl-otel
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-otel'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-otel'
+Provides-Extra: etl-polars
+Requires-Dist: deltalake<2.0,>=1.5; extra == 'etl-polars'
+Requires-Dist: fsspec>=2024.2.0; extra == 'etl-polars'
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-polars'
+Requires-Dist: polars<2.0,>=1.0; extra == 'etl-polars'
+Provides-Extra: etl-spark
+Requires-Dist: delta-spark<4.0,>=3.2; extra == 'etl-spark'
+Requires-Dist: findspark<3.0,>=2.0; extra == 'etl-spark'
+Requires-Dist: fsspec>=2024.2.0; extra == 'etl-spark'
+Requires-Dist: opentelemetry-api<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-spark'
+Requires-Dist: pyspark<4.0,>=3.5; extra == 'etl-spark'
 Provides-Extra: prometheus
 Requires-Dist: prometheus-client>=0.20; extra == 'prometheus'
 Provides-Extra: pyspark
@@ -33,11 +56,9 @@ Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy<3.0,>=2.0; extra == 'sqlalchemy'
 Description-Content-Type: text/markdown
 
-<div style="width:100%;background:#fff;padding:4px 0;box-sizing:border-box;">
-  <p align="center" style="margin:0;">
-    <img src="docs/_static/logo-transparent.png" alt="loom-kernel" style="display:block;width:100%;max-width:none;max-height:220px;height:auto;object-fit:contain;margin:0 auto;" />
-  </p>
-</div>
+<p align="center">
+  <img src="docs/_static/logo-transparent.png" alt="loom-kernel" width="160" style="background:#ffffff;border-radius:6px;padding:8px 20px;" />
+</p>
 
 # loom-kernel
 
@@ -72,7 +93,8 @@ infrastructure (DB, cache, transport) without breaking business logic.
 
 - Usage guides and architecture docs are available in the `docs/` site.
 - API reference is autogenerated from public docstrings.
-- End-to-end demo application: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end REST demo: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end ETL demo: [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl) — full Polars and Spark pipeline examples.
 
 ## Main subpaths
 
@@ -84,10 +106,23 @@ infrastructure (DB, cache, transport) without breaking business logic.
 | `src/loom/core/repository/sqlalchemy` | Concrete async SQLAlchemy repository implementation. |
 | `src/loom/core/model` | Base model, fields, relations, and entity introspection. |
 | `src/loom/core/cache` | Decorators and cached repository with dependency invalidation. |
+| `src/loom/core/config` | YAML config loader with cloud URI support and pluggable resolvers. |
 | `src/loom/rest` | Framework-agnostic REST model and route compiler. |
 | `src/loom/rest/fastapi` | Direct FastAPI integration (auto wiring and runtime router). |
 | `src/loom/prometheus` | Middleware and adapter for runtime metrics. |
 | `src/loom/testing` | Harnesses for unit/integration tests and golden tests. |
+| `src/loom/etl` | Declarative ETL subsystem — pipelines, Polars/Spark backends, observability. |
+| `src/loom/etl/pipeline` | `ETLStep`, `ETLProcess`, `ETLPipeline`, `StepSQL`, and `ETLParams`. |
+| `src/loom/etl/declarative` | `FromTable`, `FromFile`, `IntoTable`, `IntoFile`, predicate DSL, and `params` proxy. |
+| `src/loom/etl/schema` | Backend-agnostic schema model (`LoomDtype`, `ColumnSchema`, `TableRef`). |
+| `src/loom/etl/storage` | Storage config, table/file locators, and route resolution. |
+| `src/loom/etl/compiler` | Compile-time validation and execution plan builder. |
+| `src/loom/etl/runner` | `ETLRunner` entry point and YAML config loader. |
+| `src/loom/etl/checkpoint` | Step-level checkpoint store for incremental re-runs. |
+| `src/loom/etl/observability` | Run/step observers, OTEL sink, structlog sink, and execution records. |
+| `src/loom/etl/backends/polars` | Polars + Delta Lake reader, writer, and schema aligner. |
+| `src/loom/etl/backends/spark` | Spark + Delta reader, writer, and schema aligner. |
+| `src/loom/etl/testing` | `PolarsStepRunner`, `SparkStepRunner`, `ETLScenario`, and test stubs. |
 
 ## Quick start
 

{loom_kernel-0.2.0 → loom_kernel-0.3.0}/README.md

@@ -1,8 +1,6 @@
-<div style="width:100%;background:#fff;padding:4px 0;box-sizing:border-box;">
-  <p align="center" style="margin:0;">
-    <img src="docs/_static/logo-transparent.png" alt="loom-kernel" style="display:block;width:100%;max-width:none;max-height:220px;height:auto;object-fit:contain;margin:0 auto;" />
-  </p>
-</div>
+<p align="center">
+  <img src="docs/_static/logo-transparent.png" alt="loom-kernel" width="160" style="background:#ffffff;border-radius:6px;padding:8px 20px;" />
+</p>
 
 # loom-kernel
 
@@ -37,7 +35,8 @@ infrastructure (DB, cache, transport) without breaking business logic.
 
 - Usage guides and architecture docs are available in the `docs/` site.
 - API reference is autogenerated from public docstrings.
-- End-to-end demo application: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end REST demo: [`dummy-loom`](https://github.com/the-reacher-data/dummy-loom).
+- End-to-end ETL demo: [`dummy-loom-etl`](https://github.com/the-reacher-data/dummy-loom-etl) — full Polars and Spark pipeline examples.
 
 ## Main subpaths
 
@@ -49,10 +48,23 @@ infrastructure (DB, cache, transport) without breaking business logic.
 | `src/loom/core/repository/sqlalchemy` | Concrete async SQLAlchemy repository implementation. |
 | `src/loom/core/model` | Base model, fields, relations, and entity introspection. |
 | `src/loom/core/cache` | Decorators and cached repository with dependency invalidation. |
+| `src/loom/core/config` | YAML config loader with cloud URI support and pluggable resolvers. |
 | `src/loom/rest` | Framework-agnostic REST model and route compiler. |
 | `src/loom/rest/fastapi` | Direct FastAPI integration (auto wiring and runtime router). |
 | `src/loom/prometheus` | Middleware and adapter for runtime metrics. |
 | `src/loom/testing` | Harnesses for unit/integration tests and golden tests. |
+| `src/loom/etl` | Declarative ETL subsystem — pipelines, Polars/Spark backends, observability. |
+| `src/loom/etl/pipeline` | `ETLStep`, `ETLProcess`, `ETLPipeline`, `StepSQL`, and `ETLParams`. |
+| `src/loom/etl/declarative` | `FromTable`, `FromFile`, `IntoTable`, `IntoFile`, predicate DSL, and `params` proxy. |
+| `src/loom/etl/schema` | Backend-agnostic schema model (`LoomDtype`, `ColumnSchema`, `TableRef`). |
+| `src/loom/etl/storage` | Storage config, table/file locators, and route resolution. |
+| `src/loom/etl/compiler` | Compile-time validation and execution plan builder. |
+| `src/loom/etl/runner` | `ETLRunner` entry point and YAML config loader. |
+| `src/loom/etl/checkpoint` | Step-level checkpoint store for incremental re-runs. |
+| `src/loom/etl/observability` | Run/step observers, OTEL sink, structlog sink, and execution records. |
+| `src/loom/etl/backends/polars` | Polars + Delta Lake reader, writer, and schema aligner. |
+| `src/loom/etl/backends/spark` | Spark + Delta reader, writer, and schema aligner. |
+| `src/loom/etl/testing` | `PolarsStepRunner`, `SparkStepRunner`, `ETLScenario`, and test stubs. |
 
 ## Quick start
 

loom_kernel-0.3.0/docs/_static/custom.css

@@ -0,0 +1,13 @@
+.loom-hero {
+  padding: 1rem 0 1.5rem;
+  text-align: center;
+}
+
+.loom-hero img {
+  display: inline-block;
+  width: 160px;
+  height: auto;
+  background: #ffffff;
+  border-radius: 6px;
+  padding: 8px 20px;
+}

{loom_kernel-0.2.0 → loom_kernel-0.3.0}/docs/conf.py

@@ -6,6 +6,7 @@ import os
 import sys
 from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 ROOT = Path(__file__).resolve().parents[1]
 SRC = ROOT / "src"
@@ -52,7 +53,14 @@ napoleon_google_docstring = True
 napoleon_numpy_docstring = False
 
 
-def _skip_duplicate_reexports(app, what, name, obj, skip, options):
+def _skip_duplicate_reexports(
+    app: Any,
+    what: str,
+    name: str,
+    obj: Any,
+    skip: bool,
+    options: Any,
+) -> bool:
     """Skip known re-exported symbols that duplicate canonical API objects."""
     del what, obj, options
     current_module = app.env.temp_data.get("autodoc:module")
@@ -80,16 +88,24 @@ intersphinx_mapping = {
 # Optional dependencies are mocked to keep docs builds lightweight and stable.
 autodoc_mock_imports = [
     "aiocache",
+    "celery",
+    "deltalake",
     "fastapi",
+    "fsspec",
+    "kombu",
     "omegaconf",
+    "polars",
+    "pyarrow",
     "prometheus_client",
     "pydantic",
     "pyspark",
+    "redis",
     "sqlalchemy",
+    "starlette",
     "uvicorn",
 ]
 
 
-def setup(app):
+def setup(app: Any) -> None:
     """Register Sphinx hooks."""
     app.connect("autodoc-skip-member", _skip_duplicate_reexports)