loom-kernel 0.4.0__tar.gz → 0.5.0__tar.gz

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/.github/workflows/ci-main.yml

@@ -88,7 +88,7 @@ jobs:
           uv export --frozen --only-group dev --no-hashes --no-emit-project \
             -o "$MANIFEST_DIR/requirements-dev.txt"
 
-          for extra in rest sqlalchemy config cache pyspark prometheus celery; do
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
             uv export --frozen --no-default-groups --no-dev --extra "$extra" \
               --no-hashes --no-emit-project \
               -o "$MANIFEST_DIR/requirements-extra-${extra}.txt"
@@ -103,7 +103,7 @@ jobs:
             --file="$MANIFEST_DIR/requirements-dev.txt" --package-manager=pip \
             --project-name="loom-py/dev" --target-reference="${GITHUB_REF_NAME:-master}"
 
-          for extra in rest sqlalchemy config cache pyspark prometheus celery; do
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
             python -m pip install -r "$MANIFEST_DIR/requirements-extra-${extra}.txt"
             snyk monitor --skip-unresolved \
               --file="$MANIFEST_DIR/requirements-extra-${extra}.txt" \

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/.github/workflows/ci-pr.yml

@@ -142,7 +142,7 @@ jobs:
           uv export --frozen --only-group dev --no-hashes --no-emit-project \
             -o "$MANIFEST_DIR/requirements-dev.txt"
 
-          for extra in rest sqlalchemy config cache pyspark prometheus celery; do
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
             uv export --frozen --no-default-groups --no-dev --extra "$extra" \
               --no-hashes --no-emit-project \
               -o "$MANIFEST_DIR/requirements-extra-${extra}.txt"
@@ -162,7 +162,7 @@ jobs:
             --package-manager=pip \
             --project-name="loom-py/dev"
 
-          for extra in rest sqlalchemy config cache pyspark prometheus celery; do
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
             python -m pip install -r "$MANIFEST_DIR/requirements-extra-${extra}.txt"
             snyk test --severity-threshold=high \
               --skip-unresolved \

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/.gitignore

@@ -16,6 +16,7 @@ wheels/
 coverage.xml
 htmlcov/
 .vscode/
+.pi/
 docs/__pycache__/
 .claude/
 .claude.md

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/CHANGELOG.md

@@ -1,3 +1,30 @@
+# 🚀 Release 0.5.0 ([#20](https://github.com/the-reacher-data/loom-py/pull/20)) ([`70f7cf8`](https://github.com/the-reacher-data/loom-py/commit/70f7cf85275fcc1f590e06f980dfac91cad50893))
+
+
+## ✨ Features
+### core
+- **core:** add shared expression routing primitives
+
+### streaming
+- **streaming:** add process DSL and routing
+- **streaming:** add kafka transport and observability
+- **streaming:** allow task resource kwargs
+- **streaming:** add scoped dependency nodes
+- **streaming:** make tasks configurable
+
+### config
+- **config:** add declarative config bindings
+
+
+
+
+
+
+
+
+
+
+
 # 🚀 Release 0.4.0 ([#18](https://github.com/the-reacher-data/loom-py/pull/18)) ([`09e5aa3`](https://github.com/the-reacher-data/loom-py/commit/09e5aa340f88e0488daca2bf10320aad7aebbe1d))
 
 

loom_kernel-0.5.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,25 @@
+# 🚀 Release 0.5.0 ([#20](https://github.com/the-reacher-data/loom-py/pull/20)) ([`70f7cf8`](https://github.com/the-reacher-data/loom-py/commit/70f7cf85275fcc1f590e06f980dfac91cad50893))
+
+
+## ✨ Features
+### core
+- **core:** add shared expression routing primitives
+
+### streaming
+- **streaming:** add process DSL and routing
+- **streaming:** add kafka transport and observability
+- **streaming:** allow task resource kwargs
+- **streaming:** add scoped dependency nodes
+- **streaming:** make tasks configurable
+
+### config
+- **config:** add declarative config bindings
+
+
+
+
+
+
+
+
+

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loom-kernel
-Version: 0.4.0
+Version: 0.5.0
 Summary: Loom Python project
 License-File: LICENSE
 Requires-Python: >=3.11
@@ -54,6 +54,15 @@ Requires-Dist: uvicorn<1.0,>=0.30; extra == 'rest'
 Requires-Dist: uvloop<1.0,>=0.21; (sys_platform != 'win32') and extra == 'rest'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy<3.0,>=2.0; extra == 'sqlalchemy'
+Provides-Extra: streaming
+Requires-Dist: anyio<5.0,>=4.0; extra == 'streaming'
+Requires-Dist: bytewax>=0.21; extra == 'streaming'
+Requires-Dist: confluent-kafka<3.0,>=2.6; extra == 'streaming'
+Requires-Dist: opentelemetry-api<2.0,>=1.27; extra == 'streaming'
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.27; extra == 'streaming'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.27; extra == 'streaming'
+Requires-Dist: opentelemetry-sdk<2.0,>=1.27; extra == 'streaming'
+Requires-Dist: uvloop<1.0,>=0.21; (sys_platform != 'win32') and extra == 'streaming'
 Description-Content-Type: text/markdown
 
 <p align="center">

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/docs/conf.py

@@ -69,6 +69,8 @@ def _skip_duplicate_reexports(
         ("loom.core.errors", "RuleViolations"),
         ("loom.core.use_case", "RuleViolation"),
         ("loom.core.use_case", "RuleViolations"),
+        ("loom.etl.observability", "OtelConfig"),
+        ("loom.testing", "CompilationError"),
         ("loom.rest.model", "PaginationMode"),
     }
     if (current_module, name) in duplicated_reexports:

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/docs/getting-started/rest.md

@@ -1,4 +1,4 @@
-# Quickstart
+# REST Quickstart
 
 `loom-kernel` lets you define business behavior in typed use cases and expose it
 through REST interfaces — with full auto-CRUD, background jobs, and Celery workers.

loom_kernel-0.5.0/docs/getting-started/streaming.md

@@ -0,0 +1,97 @@
+# Streaming Quickstart
+
+`loom.streaming` lets you declare typed streaming flows with explicit sources,
+steps, batching, branching, and transport wiring.
+
+The companion demo repository for this subsystem is
+`dummy-loom-streaming <https://github.com/the-reacher-data/dummy-loom-streaming>`_.
+
+## Install
+
+Choose the streaming runtime and Kafka transport:
+
+```bash
+pip install "loom-kernel[streaming,kafka]"
+```
+
+## Minimal flow
+
+```python
+from loom.streaming import (
+    CollectBatch,
+    FromTopic,
+    IntoTopic,
+    Process,
+    StreamFlow,
+    WithAsync,
+    payload,
+)
+
+from app.streaming.tasks import FetchRequestTask
+from app.streaming.types import ScrapeRequest, ScrapeResponse
+
+
+async_scrape_flow = StreamFlow(
+    name="async_smoke_flow",
+    source=FromTopic[ScrapeRequest](
+        name="scrape.requests",
+        payload=ScrapeRequest,
+    ),
+    process=Process(
+        CollectBatch(max_records=50, timeout_ms=2000),
+        WithAsync(
+            process=Process(
+                FetchRequestTask,
+                IntoTopic[ScrapeResponse](
+                    name="scrape.responses",
+                    payload=ScrapeResponse,
+                ),
+            ),
+            max_concurrency=50,
+        ),
+    ),
+)
+```
+
+## Run it
+
+Use the Bytewax runner to compile the flow, resolve YAML bindings, and execute
+the graph:
+
+```python
+from loom.streaming.bytewax import StreamingRunner
+
+runner = StreamingRunner.from_yaml(async_scrape_flow, "config/streaming.yaml")
+runner.run()
+```
+
+## Public imports
+
+Prefer the public surface:
+
+```python
+from loom.streaming import (
+    Broadcast,
+    CollectBatch,
+    ContextFactory,
+    Fork,
+    FromTopic,
+    IntoTopic,
+    Message,
+    Process,
+    RecordStep,
+    Router,
+    StreamFlow,
+    With,
+    WithAsync,
+    msg,
+    payload,
+)
+```
+
+Use `loom.streaming.nodes`, `loom.streaming.kafka`, and
+`loom.streaming.bytewax` only when you need a narrower or runtime-specific
+surface.
+
+For a runnable end-to-end example, see the companion repository:
+`dummy-loom-streaming <https://github.com/the-reacher-data/dummy-loom-streaming>`_.

loom_kernel-0.5.0/docs/guides/quickstart.md

@@ -0,0 +1,7 @@
+# REST Quickstart
+
+The canonical REST quickstart lives in
+[getting-started/rest](../getting-started/rest.md).
+
+Use the REST quickstart for model-driven APIs, auto-CRUD, use cases, jobs, and
+FastAPI delivery.

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/docs/index.rst

@@ -42,12 +42,19 @@ Official hosted documentation:
 Companion demo application:
 `dummy-loom <https://github.com/the-reacher-data/dummy-loom>`_.
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Overview
+
+   overview
+
 .. toctree::
    :maxdepth: 2
    :caption: Getting Started
 
    getting-started/rest
    getting-started/etl
+   getting-started/streaming
 
 .. toctree::
    :maxdepth: 2
@@ -67,6 +74,12 @@ Companion demo application:
    etl/testing
    etl/examples
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Streaming
+
+   streaming/bytewax
+
 .. toctree::
    :maxdepth: 2
    :caption: Reference

loom_kernel-0.5.0/docs/overview.md

@@ -0,0 +1,122 @@
+# Project Overview
+
+`loom-kernel` is a framework kernel for building backend systems with a common
+set of typed building blocks:
+
+- **Core** for models, commands, use cases, routing, logging, and shared
+  infrastructure contracts.
+- **REST** for HTTP APIs, auto-CRUD, jobs, and FastAPI-based delivery.
+- **ETL** for declarative batch pipelines with backend-specific runners.
+- **Streaming** for typed topic-oriented flows with Kafka transport and a
+  Bytewax runtime adapter.
+
+The three public subsystems solve different problems, but they share the same
+design rules:
+
+- author with typed Python objects
+- resolve configuration from YAML
+- keep runtime concerns behind adapters
+- prefer explicit contracts over reflection-heavy magic
+
+The companion demo repository for the streaming subsystem is
+`dummy-loom-streaming <https://github.com/the-reacher-data/dummy-loom-streaming>`_.
+
+## At a glance
+
+| Area | What it is | Main runtime |
+|------|------------|--------------|
+| Core | Shared kernel for models, commands, use cases, routing, logging, config, and engine primitives. | Internal kernel services |
+| REST | Request/response application layer with typed use cases, auto-CRUD, jobs, and callbacks. | FastAPI + Celery |
+| ETL | Declarative batch pipelines over tables and files, with backend-specific execution. | Polars + Spark |
+| Streaming | Typed topic-oriented flows for microservices and event-driven processing. | Kafka + Bytewax |
+
+## REST
+
+REST is the request/response part of the kernel.
+
+Use it when you want:
+
+- typed use cases with declarative dependencies
+- model-driven auto-CRUD
+- background jobs and callbacks
+- FastAPI delivery with minimal ceremony
+
+The public entrypoint is `loom.rest`, and the quickstart is focused on models,
+interfaces, and use-case wiring.
+
+REST also covers asynchronous/background work through Celery jobs and callbacks.
+
+## ETL
+
+ETL is the batch-processing part of the kernel.
+
+Use it when you want:
+
+- typed ETL parameters
+- declarative pipeline/process/step graphs
+- backend-specific execution on Polars or Spark
+- file and table targets with explicit write modes
+
+The public entrypoint is `loom.etl`, and the quickstart is focused on
+`ETLStep`, `ETLProcess`, and `ETLPipeline`.
+
+The two supported execution paths are:
+
+- Polars for local and lake-style batch processing
+- Spark for distributed batch processing
+
+The model is table-centric: sources, targets, schemas, and write modes are the
+core abstractions.
+
+## Streaming
+
+Streaming is the topic-oriented part of the kernel.
+
+Use it when you want:
+
+- typed `StreamFlow` declarations
+- Kafka topics as sources and sinks
+- synchronous and asynchronous steps
+- fan-out, routing, and batch collection
+- execution through the Bytewax adapter
+
+The public entrypoint is `loom.streaming`, with transport support in
+`loom.streaming.kafka` and runtime support in `loom.streaming.bytewax`.
+
+Streaming is the right fit for microservices, fan-out, branching, and
+message-driven workflows.
+
+The design goal is consistent authoring across domains:
+
+- declare behavior in Python types
+- resolve configuration from YAML
+- keep runtime adapters separate from authoring APIs
+- preserve strong typing and explicit contracts
+
+## How the parts fit together
+
+| Part | Authoring surface | Runtime surface |
+|------|-------------------|-----------------|
+| Core | `loom.core.*` | shared kernel services |
+| REST | `loom.rest.*` | FastAPI adapters |
+| ETL | `loom.etl.*` | Polars / Spark runners |
+| Streaming | `loom.streaming.*` | Kafka / Bytewax adapters |
+
+## Streaming in practice
+
+Streaming is split into three public layers:
+
+1. `loom.streaming` for flow declarations.
+2. `loom.streaming.kafka` for transport contracts and codecs.
+3. `loom.streaming.bytewax` for the execution adapter.
+
+The public flow authoring API uses `StreamFlow`, `Process`, `FromTopic`,
+`IntoTopic`, `CollectBatch`, `With`, `WithAsync`, `Fork`, `Broadcast`, and
+`Router`.
+
+## Documentation structure
+
+- [REST Quickstart](getting-started/rest)
+- [ETL Quickstart](getting-started/etl)
+- [Streaming Quickstart](getting-started/streaming)
+- [Streaming Bytewax runtime](streaming/bytewax)

loom_kernel-0.5.0/docs/reference/api/streaming.rst

@@ -0,0 +1,29 @@
+Streaming API
+=============
+
+Public authoring API
+--------------------
+
+.. autosummary::
+   :toctree: generated
+
+   loom.streaming
+   loom.streaming.compiler
+
+Runtime adapter
+---------------
+
+.. autosummary::
+   :toctree: generated
+
+   loom.streaming.bytewax
+   loom.streaming.bytewax.runner
+
+Transport and observability
+---------------------------
+
+.. autosummary::
+   :toctree: generated
+
+   loom.streaming.kafka
+   loom.streaming.testing

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/docs/reference/index.rst

@@ -9,6 +9,7 @@ autodoc output easier to navigate.
 
    api/core
    api/etl
+   api/streaming
    api/repository
    api/rest
    api/testing

loom_kernel-0.5.0/docs/streaming/bytewax.md

@@ -0,0 +1,62 @@
+# Bytewax Runtime
+
+`loom.streaming.bytewax` is the production adapter that turns a declarative
+`StreamFlow` into a real Bytewax dataflow.
+
+For a runnable end-to-end reference implementation, see:
+`dummy-loom-streaming <https://github.com/the-reacher-data/dummy-loom-streaming>`_.
+
+## What it does
+
+- compiles the flow
+- resolves config bindings
+- wires Kafka sources and sinks
+- applies `With` / `WithAsync`
+- runs observability hooks
+
+## Public entrypoint
+
+```python
+from loom.streaming.bytewax import StreamingRunner
+```
+
+## When to use it
+
+Use the Bytewax adapter when you want:
+
+- topic-to-topic streaming
+- typed message envelopes
+- branch-aware fan-out
+- `CollectBatch` before `WithAsync`
+- runtime observability for batch and node lifecycle events
+
+## Flow contract
+
+The clean authoring contract is:
+
+```python
+from loom.streaming import CollectBatch, IntoTopic, Process, WithAsync
+
+process = Process(
+    CollectBatch(max_records=50, timeout_ms=2000),
+    WithAsync(
+        process=Process(
+            # per-message async step
+            ...
+            IntoTopic(...),
+        ),
+        max_concurrency=50,
+    ),
+)
+```
+
+`CollectBatch` belongs before `WithAsync` when you want batch aggregation as an
+input shape. If you want to wait for a batch outside the async step, keep that
+logic in the parent flow, not inside the task itself.
+
+## Runtime notes
+
+- `WithAsync` executes one message per task, concurrently.
+- `CollectBatch` emits observable batch events.
+- Errors are routed through the explicit error wiring.
+- `msg` and `payload` are public expression roots for routing predicates.

{loom_kernel-0.4.0 → loom_kernel-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "loom-kernel"
-version = "0.4.0"
+version = "0.5.0"
 description = "Loom Python project"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -63,6 +63,16 @@ pyspark = [
 prometheus = [
     "prometheus-client>=0.20",
 ]
+streaming = [
+    "anyio>=4.0,<5.0",
+    "confluent-kafka>=2.6,<3.0",
+    "bytewax>=0.21",
+    "uvloop>=0.21,<1.0; sys_platform != 'win32'",
+    "opentelemetry-api>=1.27,<2.0",
+    "opentelemetry-sdk>=1.27,<2.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.27,<2.0",
+    "opentelemetry-exporter-otlp-proto-grpc>=1.27,<2.0",
+]
 celery = [
     "celery>=5.3,<6.0",
     "kombu>=5.3,<6.0",
@@ -113,6 +123,8 @@ dev = [
     "chispa>=0.10,<1.0",
     "pytest-cov>=7.1.0",
     "findspark>=2.0.1",
+    "confluent-kafka>=2.6,<3.0",
+    "bytewax>=0.21",
     "opentelemetry-api>=1.20,<2.0",
     "opentelemetry-sdk>=1.20,<2.0",
     "opentelemetry-exporter-otlp-proto-http>=1.20,<2.0",
@@ -202,10 +214,16 @@ asyncio_mode = "auto"
 log_cli = true
 log_cli_level = "DEBUG"
 log_cli_format = "%(asctime)s %(levelname)s %(name)s %(message)s"
+markers = [
+    "bytewax: tests that exercise the Bytewax adapter and runtime wiring",
+    "kafka: tests that exercise Kafka transport and codecs",
+    "integration: end-to-end streaming flow tests",
+    "slow: long-running tests that should be opt-in in CI",
+]
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.4.0"
+version = "0.5.0"
 tag_format = "v$version"
 version_files = [
     "pyproject.toml:project.version",