loom-kernel 0.7.0__tar.gz → 0.8.0__tar.gz

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/.github/workflows/release.yml

@@ -87,7 +87,7 @@ jobs:
           sed -i "0,/^version = \".*\"/s//version = \"${VERSION}\"/" pyproject.toml
           sed -i "/^\[tool.commitizen\]/,/^\[/ s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
 
-      - name: Open version bump PR
+      - name: Create or merge version bump PR
         if: ${{ steps.version.outputs.deploy == 'true' }}
         env:
           VERSION: ${{ steps.version.outputs.version }}
@@ -99,22 +99,36 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
           BRANCH="docs/release-v${VERSION}"
-          git checkout -B "${BRANCH}"
 
-          if [ -f CHANGELOG_RELEASE.md ]; then
-            HEADER=$(head -1 CHANGELOG_RELEASE.md)
-            if ! grep -qF "${HEADER}" CHANGELOG.md; then
-              cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
-              mv CHANGELOG_MERGED.md CHANGELOG.md
-            fi
+          if git ls-remote --tags origin "refs/tags/v${VERSION}" | grep -q .; then
+            echo "Release tag v${VERSION} already exists; skipping version bump PR."
+            exit 0
+          fi
+
+          open_pr_count=$(gh pr list --head "${BRANCH}" --state open --json number --jq 'length')
+          all_pr_count=$(gh pr list --head "${BRANCH}" --state all --json number --jq 'length')
+
+          if [ "${all_pr_count}" -gt 0 ] && [ "${open_pr_count}" -eq 0 ]; then
+            echo "Release PR ${BRANCH} already exists in a closed state; skipping duplicate creation."
+            exit 0
           fi
 
-          git add pyproject.toml CHANGELOG.md
-          git diff --staged --quiet && exit 0
-          git commit -m "chore(release): bump version to ${VERSION}"
-          git push origin "${BRANCH}" --force
+          if [ "${open_pr_count}" -eq 0 ]; then
+            git checkout -B "${BRANCH}"
+
+            if [ -f CHANGELOG_RELEASE.md ]; then
+              HEADER=$(head -1 CHANGELOG_RELEASE.md)
+              if ! grep -qF "${HEADER}" CHANGELOG.md; then
+                cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
+                mv CHANGELOG_MERGED.md CHANGELOG.md
+              fi
+            fi
+
+            git add pyproject.toml CHANGELOG.md
+            git diff --staged --quiet && exit 0
+            git commit -m "chore(release): bump version to ${VERSION}"
+            git push origin "${BRANCH}" --force
 
-          if ! gh pr list --head "${BRANCH}" --state open --json number --jq '.[0].number' | grep -qE '^[0-9]+$'; then
             gh pr create \
               --title "chore(release): bump version to ${VERSION}" \
               --body "Automated version bump and changelog update after release v${VERSION}." \
@@ -136,10 +150,13 @@ jobs:
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
         with:
           password: ${{ env.PYPI_API_TOKEN }}
+          skip-existing: true
 
       - name: Publish package to PyPI (trusted publisher)
         if: ${{ steps.version.outputs.deploy == 'true' && env.PYPI_API_TOKEN == '' }}
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+        with:
+          skip-existing: true
 
       - name: Create and push tags
         if: ${{ steps.version.outputs.deploy == 'true' }}

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/CHANGELOG.md

@@ -1,3 +1,112 @@
+# 🚀 Release 0.8.0 ([#30](https://github.com/the-reacher-data/loom-py/pull/30)) ([`702b99e`](https://github.com/the-reacher-data/loom-py/commit/702b99ec75a4130f273322f8eb488637a2f98a14))
+
+
+## ✨ Features
+### streaming
+- **streaming:** add IntoSink/SinkPartition protocols and Decompose node<br>
+  > Introduces the two-level contract that makes storage sinks extensible at
+  > the node level. Any frozen dataclass satisfying IntoSink is a first-class
+  > terminal node — no registration, no inheritance, no framework coupling.
+  > core/schema/: new shared module; SchemaMode promoted from ETL (ETL re-exports for backwards compat)
+  > nodes/_sink.py: SinkPartition (contravariant per-worker protocol) and IntoSink (runtime-checkable terminal protocol)
+  > nodes/_decompose.py: EntityDecomposer protocol and Decompose transformation node
+  > validate.py: _is_leaf_terminal() helper unifies all terminal checks; IntoSink recognised as terminal in shape and output validation; _node_output_shape refactored to dispatch map
+  > exports: IntoSink, SinkPartition, Decompose, EntityDecomposer in public API
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** add IntoTable sink node and rename EntityDecomposer to PayloadExpander<br>
+  > IntoTable: frozen dataclass implementing IntoSink for SQLAlchemy and Delta backends
+  > Backend enum: SQLALCHEMY and DELTA variants
+  > _SQLAlchemyTablePartition: bulk-insert via engine.begin() per epoch batch
+  > _DeltaTablePartition: write via deltalake + polars with validated write mode
+  > Rename EntityDecomposer -> PayloadExpander, decompose() -> expand(), targets -> outputs
+  > Update both streaming.__init__ and streaming.nodes.__init__ public exports
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/compiler
+- **streaming/compiler:** teach compiler to resolve IntoSink nodes<br>
+  > Add CompiledStorageSink to _plan: holds the IntoSink node and its
+  > pre-fetched streaming.sinks.<name> config section
+  > Add terminal_storage_sinks field to CompiledPlan (default empty dict
+  > keeps all existing tests green)
+  > Refactor _build_terminal_sinks and all branch builders to return
+  > (kafka_sinks, storage_sinks) tuple in a single walk — no duplicate
+  > traversal
+  > Add _build_storage_sink: resolves config by node.name or passes {}
+  > when name is empty (self-configured sinks)
+  > Add validate_storage_sinks phase: reports a clear error for every
+  > named IntoSink whose streaming.sinks.<name> section is absent
+  > Wire validate_storage_sinks into the compiler pipeline
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming/compiler:** add Decompose shape and structural validation<br>
+  > Import Decompose in validate.py
+  > Add RECORD to _node_input_shape for Decompose — enforces that it
+  > receives individual events, not batches
+  > Add Decompose: StreamShape.RECORD to _FIXED_OUTPUT_SHAPES — its
+  > output feeds the Router as per-type records
+  > Add structural check in _validate_shape_sequence: Decompose must be
+  > immediately followed by a Router; clear error message when it is not
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/bytewax
+- **streaming/bytewax:** wire IntoSink nodes to Bytewax output operators<br>
+  > Add handlers/storage.py: _StorageSinkPartition wraps loom SinkPartition,
+  > extracting payload from Message envelopes; _StorageDynamicSink calls
+  > node.build_partition(config, worker_index, worker_count) once per
+  > Bytewax worker at startup; _apply_into_sink registers the DynamicSink
+  > via bw_output for any IntoSink node found in the compiled plan
+  > Register IntoSink in _NODE_HANDLERS dispatch map (dispatcher.py) so
+  > _wire_process routes any IntoSink node to _apply_into_sink
+  > Add IntoSink pass-through in _execute_router_node (routing.py)
+  > consistent with the IntoTopic/Drain pattern for inline Router execution
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
+
+
+## ♻️ Refactor
+### streaming
+- **streaming:** improve _SQLAlchemyTablePartition with typed columns and pool config<br>
+  > Extract _require_sa_url, _sa_engine_kwargs, _sa_type_for, _sa_table_from_struct,
+  > _structs_to_rows helpers for readability and single responsibility
+  > Map Python types to SA column types (_sa_type_for) with Optional unwrapping
+  > and collection → JSON fallback; mirrors _SCALAR_TYPE_MAP from introspection.py
+  > Mirror SessionManager pool defaults: pool_pre_ping, echo, pool_size,
+  > max_overflow, pool_timeout, pool_recycle, connect_args from config
+  > Replace msgspec.structs.asdict cast hack with direct call via _structs_to_rows
+  > Drop connection_string fallback — config contract is url only
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** merge sink config from dsl and yaml
+- **streaming:** move table sink into package
+- **streaming:** resolve table sink config via context
+- **streaming:** enforce typed table sink config
+
+
+
+## ✅ Tests
+### streaming
+- **streaming:** add IntoTable SQLAlchemy integration tests against SQLite<br>
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** cover resource manager session cache
+
+
+
+## 🔖 Other
+- Fix Bytewax step flat_map tests
+- Refactor streaming validation helpers<br>
+  > --------
+  > Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
 # 🚀 Release 0.7.0 ([#25](https://github.com/the-reacher-data/loom-py/pull/25)) ([`0d8941a`](https://github.com/the-reacher-data/loom-py/commit/0d8941a724ec768b162a07e756bac3d53157ec85))
 
 

loom_kernel-0.8.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,107 @@
+# 🚀 Release 0.8.0 ([#30](https://github.com/the-reacher-data/loom-py/pull/30)) ([`702b99e`](https://github.com/the-reacher-data/loom-py/commit/702b99ec75a4130f273322f8eb488637a2f98a14))
+
+
+## ✨ Features
+### streaming
+- **streaming:** add IntoSink/SinkPartition protocols and Decompose node<br>
+  > Introduces the two-level contract that makes storage sinks extensible at
+  > the node level. Any frozen dataclass satisfying IntoSink is a first-class
+  > terminal node — no registration, no inheritance, no framework coupling.
+  > core/schema/: new shared module; SchemaMode promoted from ETL (ETL re-exports for backwards compat)
+  > nodes/_sink.py: SinkPartition (contravariant per-worker protocol) and IntoSink (runtime-checkable terminal protocol)
+  > nodes/_decompose.py: EntityDecomposer protocol and Decompose transformation node
+  > validate.py: _is_leaf_terminal() helper unifies all terminal checks; IntoSink recognised as terminal in shape and output validation; _node_output_shape refactored to dispatch map
+  > exports: IntoSink, SinkPartition, Decompose, EntityDecomposer in public API
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** add IntoTable sink node and rename EntityDecomposer to PayloadExpander<br>
+  > IntoTable: frozen dataclass implementing IntoSink for SQLAlchemy and Delta backends
+  > Backend enum: SQLALCHEMY and DELTA variants
+  > _SQLAlchemyTablePartition: bulk-insert via engine.begin() per epoch batch
+  > _DeltaTablePartition: write via deltalake + polars with validated write mode
+  > Rename EntityDecomposer -> PayloadExpander, decompose() -> expand(), targets -> outputs
+  > Update both streaming.__init__ and streaming.nodes.__init__ public exports
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/compiler
+- **streaming/compiler:** teach compiler to resolve IntoSink nodes<br>
+  > Add CompiledStorageSink to _plan: holds the IntoSink node and its
+  > pre-fetched streaming.sinks.<name> config section
+  > Add terminal_storage_sinks field to CompiledPlan (default empty dict
+  > keeps all existing tests green)
+  > Refactor _build_terminal_sinks and all branch builders to return
+  > (kafka_sinks, storage_sinks) tuple in a single walk — no duplicate
+  > traversal
+  > Add _build_storage_sink: resolves config by node.name or passes {}
+  > when name is empty (self-configured sinks)
+  > Add validate_storage_sinks phase: reports a clear error for every
+  > named IntoSink whose streaming.sinks.<name> section is absent
+  > Wire validate_storage_sinks into the compiler pipeline
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming/compiler:** add Decompose shape and structural validation<br>
+  > Import Decompose in validate.py
+  > Add RECORD to _node_input_shape for Decompose — enforces that it
+  > receives individual events, not batches
+  > Add Decompose: StreamShape.RECORD to _FIXED_OUTPUT_SHAPES — its
+  > output feeds the Router as per-type records
+  > Add structural check in _validate_shape_sequence: Decompose must be
+  > immediately followed by a Router; clear error message when it is not
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/bytewax
+- **streaming/bytewax:** wire IntoSink nodes to Bytewax output operators<br>
+  > Add handlers/storage.py: _StorageSinkPartition wraps loom SinkPartition,
+  > extracting payload from Message envelopes; _StorageDynamicSink calls
+  > node.build_partition(config, worker_index, worker_count) once per
+  > Bytewax worker at startup; _apply_into_sink registers the DynamicSink
+  > via bw_output for any IntoSink node found in the compiled plan
+  > Register IntoSink in _NODE_HANDLERS dispatch map (dispatcher.py) so
+  > _wire_process routes any IntoSink node to _apply_into_sink
+  > Add IntoSink pass-through in _execute_router_node (routing.py)
+  > consistent with the IntoTopic/Drain pattern for inline Router execution
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
+
+
+## ♻️ Refactor
+### streaming
+- **streaming:** improve _SQLAlchemyTablePartition with typed columns and pool config<br>
+  > Extract _require_sa_url, _sa_engine_kwargs, _sa_type_for, _sa_table_from_struct,
+  > _structs_to_rows helpers for readability and single responsibility
+  > Map Python types to SA column types (_sa_type_for) with Optional unwrapping
+  > and collection → JSON fallback; mirrors _SCALAR_TYPE_MAP from introspection.py
+  > Mirror SessionManager pool defaults: pool_pre_ping, echo, pool_size,
+  > max_overflow, pool_timeout, pool_recycle, connect_args from config
+  > Replace msgspec.structs.asdict cast hack with direct call via _structs_to_rows
+  > Drop connection_string fallback — config contract is url only
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** merge sink config from dsl and yaml
+- **streaming:** move table sink into package
+- **streaming:** resolve table sink config via context
+- **streaming:** enforce typed table sink config
+
+
+
+## ✅ Tests
+### streaming
+- **streaming:** add IntoTable SQLAlchemy integration tests against SQLite<br>
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** cover resource manager session cache
+
+
+
+## 🔖 Other
+- Fix Bytewax step flat_map tests
+- Refactor streaming validation helpers<br>
+  > --------
+  > Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loom-kernel
-Version: 0.7.0
+Version: 0.8.0
 Summary: Loom Python project
 License-File: LICENSE
 Requires-Python: >=3.11

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "loom-kernel"
-version = "0.7.0"
+version = "0.8.0"
 description = "Loom Python project"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -220,7 +220,7 @@ markers = [
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.7.0"
+version = "0.8.0"
 tag_format = "v$version"
 version_files = [
     "pyproject.toml:project.version",

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/src/loom/core/repository/sqlalchemy/session_manager.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from contextlib import asynccontextmanager
 from typing import Any
 
@@ -19,6 +19,46 @@ from loom.core.tracing import get_trace_id
 class SessionManager:
     """Async SQLAlchemy session manager with pooling support."""
 
+    @classmethod
+    def from_config(
+        cls,
+        config: Mapping[str, Any],
+        *,
+        inject_trace_id: bool = True,
+        **engine_kwargs: object,
+    ) -> SessionManager:
+        """Build a session manager from a resolved SQLAlchemy config mapping.
+
+        Args:
+            config: Resolved config mapping containing a ``url`` entry and
+                optional pool tuning keys.
+            inject_trace_id: When ``True``, prefixes SQL statements with the
+                active trace id when available.
+            **engine_kwargs: Additional keyword arguments forwarded to the
+                async engine constructor.
+
+        Returns:
+            A configured :class:`SessionManager`.
+
+        Raises:
+            ValueError: If ``url`` is missing or empty.
+        """
+        url = config.get("url")
+        if not url:
+            raise ValueError("SQLAlchemy sink config requires a 'url'.")
+        return cls(
+            str(url),
+            echo=bool(config.get("echo", False)),
+            pool_pre_ping=bool(config.get("pool_pre_ping", True)),
+            pool_size=_optional_int(config.get("pool_size"), 10),
+            max_overflow=_optional_int(config.get("max_overflow"), 20),
+            pool_timeout=_optional_int(config.get("pool_timeout"), 30),
+            pool_recycle=_optional_int(config.get("pool_recycle"), 1800),
+            connect_args=dict(config.get("connect_args") or {}),
+            inject_trace_id=inject_trace_id,
+            **engine_kwargs,
+        )
+
     def __init__(
         self,
         url: str,
@@ -139,3 +179,9 @@ def _register_trace_id_listener(async_engine: AsyncEngine) -> None:
         if tid:
             statement = f"/* trace_id={tid} */ " + statement
         return statement, parameters
+
+
+def _optional_int(value: Any, default: int) -> int:
+    if value is None:
+        return default
+    return int(value)

loom_kernel-0.8.0/src/loom/core/schema/__init__.py

@@ -0,0 +1,5 @@
+"""Shared schema primitives promoted from ETL for use across all loom layers."""
+
+from loom.core.schema.schema_mode import SchemaMode
+
+__all__ = ["SchemaMode"]

loom_kernel-0.8.0/src/loom/core/schema/schema_mode.py

@@ -0,0 +1,22 @@
+"""Schema evolution mode shared by ETL and streaming layers."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class SchemaMode(StrEnum):
+    """Schema evolution strategy applied before each write to a storage target.
+
+    Values:
+        STRICT:    Fail on incompatible schema changes.
+        EVOLVE:    Allow additive evolution where the backend supports it.
+        OVERWRITE: Replace the table schema with the incoming schema.
+    """
+
+    STRICT = "strict"
+    EVOLVE = "evolve"
+    OVERWRITE = "overwrite"
+
+
+__all__ = ["SchemaMode"]

loom_kernel-0.8.0/src/loom/etl/declarative/target/_schema_mode.py

@@ -0,0 +1,5 @@
+"""Schema evolution mode — re-exported from loom.core.schema for backwards compatibility."""
+
+from loom.core.schema.schema_mode import SchemaMode
+
+__all__ = ["SchemaMode"]

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/src/loom/streaming/__init__.py

@@ -31,10 +31,12 @@ from loom.streaming.nodes._boundary import (
     PartitionStrategy,
 )
 from loom.streaming.nodes._broadcast import Broadcast, BroadcastRoute
+from loom.streaming.nodes._decompose import Explode, PayloadExpander
 from loom.streaming.nodes._fork import Fork, ForkRoute
 from loom.streaming.nodes._protocols import Predicate, Selector
 from loom.streaming.nodes._router import Route, Router
 from loom.streaming.nodes._shape import CollectBatch, Drain, ForEach, StreamShape, WindowStrategy
+from loom.streaming.nodes._sink import IntoSink, SinkPartition
 from loom.streaming.nodes._step import (
     BatchExpandStep,
     BatchStep,
@@ -44,6 +46,13 @@ from loom.streaming.nodes._step import (
     Step,
     StepContext,
 )
+from loom.streaming.nodes._table import (
+    Backend,
+    DeltaSinkConfig,
+    IntoTable,
+    SqlAlchemyDatabaseConfig,
+    SqlAlchemySinkConfig,
+)
 from loom.streaming.nodes._with import (
     AsyncContextDependency,
     ContextFactory,
@@ -57,45 +66,54 @@ from loom.streaming.nodes.refs import payload as payload
 
 __all__ = [
     "AsyncContextDependency",
+    "Backend",
     "BatchExpandStep",
     "BatchStep",
     "Broadcast",
     "BroadcastRoute",
     "CollectBatch",
     "ContextFactory",
+    "DeltaSinkConfig",
+    "Explode",
     "Drain",
+    "PayloadExpander",
     "ErrorEnvelope",
     "ErrorKind",
     "ErrorMessage",
     "ErrorMessageMeta",
     "ErrorRoute",
+    "ExpandStep",
     "ForEach",
-    "FromMultiTypeTopic",
-    "FromTopic",
     "Fork",
     "ForkRoute",
-    "ExpandStep",
+    "FromMultiTypeTopic",
+    "FromTopic",
+    "IntoSink",
+    "IntoTable",
     "IntoTopic",
     "Message",
     "MessageMeta",
     "PartitionGuarantee",
     "PartitionPolicy",
     "PartitionStrategy",
+    "Predicate",
     "Process",
     "ProcessNode",
-    "Predicate",
     "RecordStep",
     "ResourceFactory",
     "ResourceScope",
     "Route",
     "Router",
     "Selector",
+    "SinkPartition",
     "StreamFlow",
     "StreamShape",
-    "WindowStrategy",
     "Step",
     "StepContext",
     "SyncContextDependency",
+    "SqlAlchemyDatabaseConfig",
+    "SqlAlchemySinkConfig",
+    "WindowStrategy",
     "With",
     "WithAsync",
     "compile_flow",

{loom_kernel-0.7.0 → loom_kernel-0.8.0}/src/loom/streaming/bytewax/_adapter.py

@@ -287,6 +287,10 @@ class _BuildContext:
         """Get or create a resource manager for *node* at position *idx*."""
         return self.resource_manager.manager_for(idx, node)
 
+    def session_manager_for(self, config: Any) -> Any:
+        """Get or create a shared SQLAlchemy session manager for one config."""
+        return self.resource_manager.session_manager_for(config)
+
     @property
     def current_path(self) -> tuple[int, ...]:
         """Return the current wiring path inside the process tree."""