loom-kernel 0.7.0__tar.gz → 0.9.0__tar.gz

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/.github/workflows/ci-main.yml

@@ -24,7 +24,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3
+      - name: Install uv
+        run: pip install uv
 
       - name: Quality report (strict)
         uses: the-reacher-data/loom-actions/actions/python/quality-report@e731576f4f9cf4ec2b36fddfc895d328c856d29f # v1

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/.github/workflows/ci-pr.yml

@@ -28,7 +28,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3
+      - name: Install uv
+        run: pip install uv
 
       - name: Configure uv environment path
         shell: bash

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/.github/workflows/docs.yml

@@ -37,7 +37,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3
+      - name: Install uv
+        run: pip install uv
 
       - name: Build docs (strict)
         shell: bash

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/.github/workflows/release.yml

@@ -37,7 +37,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3
+      - name: Install uv
+        run: pip install uv
 
       - name: Configure uv environment path
         shell: bash
@@ -87,7 +88,7 @@ jobs:
           sed -i "0,/^version = \".*\"/s//version = \"${VERSION}\"/" pyproject.toml
           sed -i "/^\[tool.commitizen\]/,/^\[/ s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
 
-      - name: Open version bump PR
+      - name: Create or merge version bump PR
         if: ${{ steps.version.outputs.deploy == 'true' }}
         env:
           VERSION: ${{ steps.version.outputs.version }}
@@ -99,22 +100,36 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
           BRANCH="docs/release-v${VERSION}"
-          git checkout -B "${BRANCH}"
 
-          if [ -f CHANGELOG_RELEASE.md ]; then
-            HEADER=$(head -1 CHANGELOG_RELEASE.md)
-            if ! grep -qF "${HEADER}" CHANGELOG.md; then
-              cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
-              mv CHANGELOG_MERGED.md CHANGELOG.md
-            fi
+          if git ls-remote --tags origin "refs/tags/v${VERSION}" | grep -q .; then
+            echo "Release tag v${VERSION} already exists; skipping version bump PR."
+            exit 0
+          fi
+
+          open_pr_count=$(gh pr list --head "${BRANCH}" --state open --json number --jq 'length')
+          all_pr_count=$(gh pr list --head "${BRANCH}" --state all --json number --jq 'length')
+
+          if [ "${all_pr_count}" -gt 0 ] && [ "${open_pr_count}" -eq 0 ]; then
+            echo "Release PR ${BRANCH} already exists in a closed state; skipping duplicate creation."
+            exit 0
           fi
 
-          git add pyproject.toml CHANGELOG.md
-          git diff --staged --quiet && exit 0
-          git commit -m "chore(release): bump version to ${VERSION}"
-          git push origin "${BRANCH}" --force
+          if [ "${open_pr_count}" -eq 0 ]; then
+            git checkout -B "${BRANCH}"
+
+            if [ -f CHANGELOG_RELEASE.md ]; then
+              HEADER=$(head -1 CHANGELOG_RELEASE.md)
+              if ! grep -qF "${HEADER}" CHANGELOG.md; then
+                cat CHANGELOG_RELEASE.md CHANGELOG.md > CHANGELOG_MERGED.md
+                mv CHANGELOG_MERGED.md CHANGELOG.md
+              fi
+            fi
+
+            git add pyproject.toml CHANGELOG.md
+            git diff --staged --quiet && exit 0
+            git commit -m "chore(release): bump version to ${VERSION}"
+            git push origin "${BRANCH}" --force
 
-          if ! gh pr list --head "${BRANCH}" --state open --json number --jq '.[0].number' | grep -qE '^[0-9]+$'; then
             gh pr create \
               --title "chore(release): bump version to ${VERSION}" \
               --body "Automated version bump and changelog update after release v${VERSION}." \
@@ -136,10 +151,13 @@ jobs:
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
         with:
           password: ${{ env.PYPI_API_TOKEN }}
+          skip-existing: true
 
       - name: Publish package to PyPI (trusted publisher)
         if: ${{ steps.version.outputs.deploy == 'true' && env.PYPI_API_TOKEN == '' }}
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+        with:
+          skip-existing: true
 
       - name: Create and push tags
         if: ${{ steps.version.outputs.deploy == 'true' }}

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/.gitignore

@@ -31,3 +31,5 @@ docs/_build/
 
 # autosummary stubs — regenerated by Sphinx on every build
 docs/reference/api/generated/
+.codegraph/
+memory/

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/CHANGELOG.md

@@ -1,3 +1,212 @@
+# 🚀 Release 0.9.0 ([#32](https://github.com/the-reacher-data/loom-py/pull/32)) ([`14072d1`](https://github.com/the-reacher-data/loom-py/commit/14072d1be9212116235f26d7a2590a4f0e622451))
+
+
+## ✨ Features
+### streaming
+- **streaming:** add Mongo CDC source
+- **streaming:** add MongoObjectId and MongoDBRef domain types<br>
+  > Replace opaque str/dict BSON normalization with structured domain types:
+  > MongoObjectId(id, created_at_ms) preserves ObjectId hex and creation
+  > timestamp extracted from the embedded 4-byte BSON timestamp field
+  > MongoDBRef(id, collection, database) models cross-collection references
+  > with explicit collection and optional database fields
+  > MongoCDCEvent.document_id widened to MongoObjectId | str | None to
+  > expose rich type when _id is an ObjectId
+  > _normalize_objectid and _normalize_dbref updated to produce the new types
+  > Guard clause added to _normalize_dbref for None identifier
+  > _message_key helper centralizes MongoObjectId → str key extraction
+  > MongoObjectId and MongoDBRef exported from loom.streaming public API
+  > Contract tests cover DBRef without database, integer id, None id error,
+  > and ObjectId without generation_time (created_at_ms=None)
+
+- **streaming:** add Backend.CLICKHOUSE with native clickhouse-connect sink<br>
+  > Add _ClickHouseTablePartition as a standalone SinkPartition backed by
+  > clickhouse-connect (HTTP/2, no SQLAlchemy). Uses a column-oriented buffer
+  > layout matching clickhouse-connect's column_oriented=True path, avoiding
+  > the internal pivot() transposition for maximum insert throughput.
+  > Resolves the _json_serializer crash that occurred when inserting list[str]
+  > fields via the SQLAlchemy backend against ClickHouseDialect_asynch. With the
+  > native client, Array(String) and all CH-native types work without workarounds.
+
+- **streaming:** add ExpandRoutes fan-out node and JsonStr semantic type<br>
+  > ExpandRoutes: expands one event once via PayloadExpander and routes each
+  > typed output list to its own independent Process (dict-based API, optional
+  > default route); wired in Bytewax via flat_map per route branch
+  > JsonStr: Annotated[str, _JsonMarker()] sentinel type for raw JSON fields;
+  > mapped to Text() in SQLAlchemy; passes through as str in ClickHouse/Delta
+  > ProcessNode union extended to include ExpandRoutes[Any]
+
+
+
+## 🐛 Fixes
+### streaming
+- **streaming:** replace assert with explicit TypeError in _adapter.py<br>
+  > Bandit B101: assert statements are stripped when running with -O.
+  > Replaced both isinstance asserts with explicit TypeError raises to
+  > preserve the runtime invariant checks under all execution modes.
+
+- **streaming:** age-based buffer flush never fired in IntoTable sink partitions<br>
+  > Two bugs prevented buffer_max_age_s from triggering a flush:
+  > 1. write_batch([]) returned immediately on empty epochs, so quiet periods
+  > never checked the age threshold.
+  > 2. _last_append_at was updated right before _should_flush(), making the
+  > elapsed-time expression always ≈0ms.
+  > Replace _last_append_at with _buffer_started_at (set when the buffer
+  > transitions from empty, reset after each flush) in both
+  > _SQLAlchemyTablePartition and _DeltaTablePartition. Empty epoch calls
+  > now check for an age-based flush before returning.
+
+- **streaming:** wire ExpandRoutes runtime and ClickHouse HTTP URL resolution
+
+### compiler
+- **compiler:** add Backend.CLICKHOUSE dispatch in build_plan storage sink resolver
+- **compiler:** validate ExpandRoutes branches for terminal output detection
+
+
+
+
+## ♻️ Refactor
+### streaming
+- **streaming:** improve Mongo CDC architecture compliance<br>
+  > Replace bare except Exception with specific OperationFailure catch
+  > Replace type(x).__name__ string dispatch with isinstance and lazy import
+  > Type watch_options as Mapping[str, object] to eliminate implicit Any
+  > Introduce _BSON_NORMALIZERS dispatch map in normalize_bson_value
+  > Define _ChangeStream and _DatabaseLike Protocols for PyMongo types
+  > Make FromMongoCDC generic with covariant MongoCDCPayloadT (PEP 696)
+  > Add needs_decode: ClassVar[bool] to compiled source types
+  > Replace isinstance type-switches with dispatch maps (_SOURCE_BUILDERS,
+  > _RUNTIME_SOURCE_BUILDERS) across compiler and bytewax runtime layers
+  > Declare typing_extensions>=4.12 as direct dependency
+  > Add contract tests for FromMongoCDC generic parametrization
+
+- **streaming:** simplify source dispatch and pymongo error handling<br>
+  > Replace _RUNTIME_SOURCE_BUILDERS dispatch map with direct isinstance check in build_runtime_source
+  > Replace lazy _operation_failure_type() with module-level try/except import for OperationFailure
+  > Use explicit isinstance in _source_topic_or_none instead of getattr duck-typing
+  > Remove redundant empty-collections branch in _open_change_stream
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
+## ✅ Tests
+### streaming
+- **streaming:** add unit tests for ExpandRoutes fan-out node
+
+
+
+
+
+# 🚀 Release 0.8.0 ([#30](https://github.com/the-reacher-data/loom-py/pull/30)) ([`702b99e`](https://github.com/the-reacher-data/loom-py/commit/702b99ec75a4130f273322f8eb488637a2f98a14))
+
+
+## ✨ Features
+### streaming
+- **streaming:** add IntoSink/SinkPartition protocols and Decompose node<br>
+  > Introduces the two-level contract that makes storage sinks extensible at
+  > the node level. Any frozen dataclass satisfying IntoSink is a first-class
+  > terminal node — no registration, no inheritance, no framework coupling.
+  > core/schema/: new shared module; SchemaMode promoted from ETL (ETL re-exports for backwards compat)
+  > nodes/_sink.py: SinkPartition (contravariant per-worker protocol) and IntoSink (runtime-checkable terminal protocol)
+  > nodes/_decompose.py: EntityDecomposer protocol and Decompose transformation node
+  > validate.py: _is_leaf_terminal() helper unifies all terminal checks; IntoSink recognised as terminal in shape and output validation; _node_output_shape refactored to dispatch map
+  > exports: IntoSink, SinkPartition, Decompose, EntityDecomposer in public API
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** add IntoTable sink node and rename EntityDecomposer to PayloadExpander<br>
+  > IntoTable: frozen dataclass implementing IntoSink for SQLAlchemy and Delta backends
+  > Backend enum: SQLALCHEMY and DELTA variants
+  > _SQLAlchemyTablePartition: bulk-insert via engine.begin() per epoch batch
+  > _DeltaTablePartition: write via deltalake + polars with validated write mode
+  > Rename EntityDecomposer -> PayloadExpander, decompose() -> expand(), targets -> outputs
+  > Update both streaming.__init__ and streaming.nodes.__init__ public exports
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/compiler
+- **streaming/compiler:** teach compiler to resolve IntoSink nodes<br>
+  > Add CompiledStorageSink to _plan: holds the IntoSink node and its
+  > pre-fetched streaming.sinks.<name> config section
+  > Add terminal_storage_sinks field to CompiledPlan (default empty dict
+  > keeps all existing tests green)
+  > Refactor _build_terminal_sinks and all branch builders to return
+  > (kafka_sinks, storage_sinks) tuple in a single walk — no duplicate
+  > traversal
+  > Add _build_storage_sink: resolves config by node.name or passes {}
+  > when name is empty (self-configured sinks)
+  > Add validate_storage_sinks phase: reports a clear error for every
+  > named IntoSink whose streaming.sinks.<name> section is absent
+  > Wire validate_storage_sinks into the compiler pipeline
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming/compiler:** add Decompose shape and structural validation<br>
+  > Import Decompose in validate.py
+  > Add RECORD to _node_input_shape for Decompose — enforces that it
+  > receives individual events, not batches
+  > Add Decompose: StreamShape.RECORD to _FIXED_OUTPUT_SHAPES — its
+  > output feeds the Router as per-type records
+  > Add structural check in _validate_shape_sequence: Decompose must be
+  > immediately followed by a Router; clear error message when it is not
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+### streaming/bytewax
+- **streaming/bytewax:** wire IntoSink nodes to Bytewax output operators<br>
+  > Add handlers/storage.py: _StorageSinkPartition wraps loom SinkPartition,
+  > extracting payload from Message envelopes; _StorageDynamicSink calls
+  > node.build_partition(config, worker_index, worker_count) once per
+  > Bytewax worker at startup; _apply_into_sink registers the DynamicSink
+  > via bw_output for any IntoSink node found in the compiled plan
+  > Register IntoSink in _NODE_HANDLERS dispatch map (dispatcher.py) so
+  > _wire_process routes any IntoSink node to _apply_into_sink
+  > Add IntoSink pass-through in _execute_router_node (routing.py)
+  > consistent with the IntoTopic/Drain pattern for inline Router execution
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
+
+
+## ♻️ Refactor
+### streaming
+- **streaming:** improve _SQLAlchemyTablePartition with typed columns and pool config<br>
+  > Extract _require_sa_url, _sa_engine_kwargs, _sa_type_for, _sa_table_from_struct,
+  > _structs_to_rows helpers for readability and single responsibility
+  > Map Python types to SA column types (_sa_type_for) with Optional unwrapping
+  > and collection → JSON fallback; mirrors _SCALAR_TYPE_MAP from introspection.py
+  > Mirror SessionManager pool defaults: pool_pre_ping, echo, pool_size,
+  > max_overflow, pool_timeout, pool_recycle, connect_args from config
+  > Replace msgspec.structs.asdict cast hack with direct call via _structs_to_rows
+  > Drop connection_string fallback — config contract is url only
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** merge sink config from dsl and yaml
+- **streaming:** move table sink into package
+- **streaming:** resolve table sink config via context
+- **streaming:** enforce typed table sink config
+
+
+
+## ✅ Tests
+### streaming
+- **streaming:** add IntoTable SQLAlchemy integration tests against SQLite<br>
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+- **streaming:** cover resource manager session cache
+
+
+
+## 🔖 Other
+- Fix Bytewax step flat_map tests
+- Refactor streaming validation helpers<br>
+  > --------
+  > Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
 # 🚀 Release 0.7.0 ([#25](https://github.com/the-reacher-data/loom-py/pull/25)) ([`0d8941a`](https://github.com/the-reacher-data/loom-py/commit/0d8941a724ec768b162a07e756bac3d53157ec85))
 
 

loom_kernel-0.9.0/CHANGELOG_RELEASE.md

@@ -0,0 +1,98 @@
+# 🚀 Release 0.9.0 ([#32](https://github.com/the-reacher-data/loom-py/pull/32)) ([`14072d1`](https://github.com/the-reacher-data/loom-py/commit/14072d1be9212116235f26d7a2590a4f0e622451))
+
+
+## ✨ Features
+### streaming
+- **streaming:** add Mongo CDC source
+- **streaming:** add MongoObjectId and MongoDBRef domain types<br>
+  > Replace opaque str/dict BSON normalization with structured domain types:
+  > MongoObjectId(id, created_at_ms) preserves ObjectId hex and creation
+  > timestamp extracted from the embedded 4-byte BSON timestamp field
+  > MongoDBRef(id, collection, database) models cross-collection references
+  > with explicit collection and optional database fields
+  > MongoCDCEvent.document_id widened to MongoObjectId | str | None to
+  > expose rich type when _id is an ObjectId
+  > _normalize_objectid and _normalize_dbref updated to produce the new types
+  > Guard clause added to _normalize_dbref for None identifier
+  > _message_key helper centralizes MongoObjectId → str key extraction
+  > MongoObjectId and MongoDBRef exported from loom.streaming public API
+  > Contract tests cover DBRef without database, integer id, None id error,
+  > and ObjectId without generation_time (created_at_ms=None)
+
+- **streaming:** add Backend.CLICKHOUSE with native clickhouse-connect sink<br>
+  > Add _ClickHouseTablePartition as a standalone SinkPartition backed by
+  > clickhouse-connect (HTTP/2, no SQLAlchemy). Uses a column-oriented buffer
+  > layout matching clickhouse-connect's column_oriented=True path, avoiding
+  > the internal pivot() transposition for maximum insert throughput.
+  > Resolves the _json_serializer crash that occurred when inserting list[str]
+  > fields via the SQLAlchemy backend against ClickHouseDialect_asynch. With the
+  > native client, Array(String) and all CH-native types work without workarounds.
+
+- **streaming:** add ExpandRoutes fan-out node and JsonStr semantic type<br>
+  > ExpandRoutes: expands one event once via PayloadExpander and routes each
+  > typed output list to its own independent Process (dict-based API, optional
+  > default route); wired in Bytewax via flat_map per route branch
+  > JsonStr: Annotated[str, _JsonMarker()] sentinel type for raw JSON fields;
+  > mapped to Text() in SQLAlchemy; passes through as str in ClickHouse/Delta
+  > ProcessNode union extended to include ExpandRoutes[Any]
+
+
+
+## 🐛 Fixes
+### streaming
+- **streaming:** replace assert with explicit TypeError in _adapter.py<br>
+  > Bandit B101: assert statements are stripped when running with -O.
+  > Replaced both isinstance asserts with explicit TypeError raises to
+  > preserve the runtime invariant checks under all execution modes.
+
+- **streaming:** age-based buffer flush never fired in IntoTable sink partitions<br>
+  > Two bugs prevented buffer_max_age_s from triggering a flush:
+  > 1. write_batch([]) returned immediately on empty epochs, so quiet periods
+  > never checked the age threshold.
+  > 2. _last_append_at was updated right before _should_flush(), making the
+  > elapsed-time expression always ≈0ms.
+  > Replace _last_append_at with _buffer_started_at (set when the buffer
+  > transitions from empty, reset after each flush) in both
+  > _SQLAlchemyTablePartition and _DeltaTablePartition. Empty epoch calls
+  > now check for an age-based flush before returning.
+
+- **streaming:** wire ExpandRoutes runtime and ClickHouse HTTP URL resolution
+
+### compiler
+- **compiler:** add Backend.CLICKHOUSE dispatch in build_plan storage sink resolver
+- **compiler:** validate ExpandRoutes branches for terminal output detection
+
+
+
+
+## ♻️ Refactor
+### streaming
+- **streaming:** improve Mongo CDC architecture compliance<br>
+  > Replace bare except Exception with specific OperationFailure catch
+  > Replace type(x).__name__ string dispatch with isinstance and lazy import
+  > Type watch_options as Mapping[str, object] to eliminate implicit Any
+  > Introduce _BSON_NORMALIZERS dispatch map in normalize_bson_value
+  > Define _ChangeStream and _DatabaseLike Protocols for PyMongo types
+  > Make FromMongoCDC generic with covariant MongoCDCPayloadT (PEP 696)
+  > Add needs_decode: ClassVar[bool] to compiled source types
+  > Replace isinstance type-switches with dispatch maps (_SOURCE_BUILDERS,
+  > _RUNTIME_SOURCE_BUILDERS) across compiler and bytewax runtime layers
+  > Declare typing_extensions>=4.12 as direct dependency
+  > Add contract tests for FromMongoCDC generic parametrization
+
+- **streaming:** simplify source dispatch and pymongo error handling<br>
+  > Replace _RUNTIME_SOURCE_BUILDERS dispatch map with direct isinstance check in build_runtime_source
+  > Replace lazy _operation_failure_type() with module-level try/except import for OperationFailure
+  > Use explicit isinstance in _source_topic_or_none instead of getattr duck-typing
+  > Remove redundant empty-collections branch in _open_change_stream
+  > Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+
+
+
+
+## ✅ Tests
+### streaming
+- **streaming:** add unit tests for ExpandRoutes fan-out node
+
+
+

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loom-kernel
-Version: 0.7.0
+Version: 0.9.0
 Summary: Loom Python project
 License-File: LICENSE
 Requires-Python: >=3.11
@@ -9,6 +9,7 @@ Requires-Dist: msgspec<1.0,>=0.18
 Requires-Dist: omegaconf<3.0,>=2.3
 Requires-Dist: prometheus-client>=0.20
 Requires-Dist: structlog<26.0,>=24.4
+Requires-Dist: typing-extensions>=4.12
 Provides-Extra: cache
 Requires-Dist: aiocache<1.0,>=0.12; extra == 'cache'
 Requires-Dist: omegaconf<3.0,>=2.3; extra == 'cache'
@@ -16,6 +17,8 @@ Provides-Extra: celery
 Requires-Dist: celery<6.0,>=5.3; extra == 'celery'
 Requires-Dist: kombu<6.0,>=5.3; extra == 'celery'
 Requires-Dist: redis<6.0,>=5.0; extra == 'celery'
+Provides-Extra: clickhouse
+Requires-Dist: clickhouse-connect<1.0,>=0.7; extra == 'clickhouse'
 Provides-Extra: config
 Requires-Dist: fsspec>=2024.2.0; extra == 'config'
 Requires-Dist: omegaconf<3.0,>=2.3; extra == 'config'
@@ -32,6 +35,7 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2.0,>=1.20; extra == 'etl-
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<2.0,>=1.20; extra == 'etl-polars'
 Requires-Dist: opentelemetry-sdk<2.0,>=1.20; extra == 'etl-polars'
 Requires-Dist: polars<2.0,>=1.0; extra == 'etl-polars'
+Requires-Dist: pyarrow>=14.0; extra == 'etl-polars'
 Provides-Extra: etl-spark
 Requires-Dist: delta-spark<4.0,>=3.2; extra == 'etl-spark'
 Requires-Dist: findspark<3.0,>=2.0; extra == 'etl-spark'

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "loom-kernel"
-version = "0.7.0"
+version = "0.9.0"
 description = "Loom Python project"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -10,6 +10,7 @@ dependencies = [
     "omegaconf>=2.3,<3.0",
     "prometheus-client>=0.20",
     "aiocache>=0.12,<1.0",
+    "typing_extensions>=4.12",
 ]
 
 [project.optional-dependencies]
@@ -24,6 +25,9 @@ rest = [
 sqlalchemy = [
     "sqlalchemy>=2.0,<3.0",
 ]
+clickhouse = [
+    "clickhouse-connect>=0.7,<1.0",
+]
 config = [
     "omegaconf>=2.3,<3.0",
     "fsspec>=2024.2.0",
@@ -35,6 +39,7 @@ cache = [
 etl-polars = [
     "polars>=1.0,<2.0",
     "deltalake>=1.5,<2.0",
+    "pyarrow>=14.0",
     "fsspec>=2024.2.0",
     "opentelemetry-api>=1.20,<2.0",
     "opentelemetry-sdk>=1.20,<2.0",
@@ -110,6 +115,8 @@ dev = [
     "ruff>=0.9.0,<1.0",
     "sqlalchemy>=2.0,<3.0",
     "polars>=1.0,<2.0",
+    "clickhouse-connect>=0.7,<1.0",
+    "pyarrow>=14.0",
     "deltalake>=1.5,<2.0",
     "fsspec>=2024.2.0",
     "httpx>=0.28.1",
@@ -122,6 +129,7 @@ dev = [
     "findspark>=2.0.1",
     "confluent-kafka>=2.6,<3.0",
     "bytewax>=0.21",
+    "pymongo>=4.6,<5.0",
     "opentelemetry-api>=1.20,<2.0",
     "opentelemetry-sdk>=1.20,<2.0",
     "opentelemetry-exporter-otlp-proto-http>=1.20,<2.0",
@@ -172,6 +180,7 @@ include = [
 typeCheckingMode = "standard"
 reportMissingModuleSource = "none"
 reportMissingImports = "none"
+reportPrivateImportUsage = "none"
 ignore = [
     "src/loom/etl/testing/spark.py",
 ]
@@ -193,9 +202,15 @@ module = [
     "findspark",
     "fsspec.*",
     "chispa.*",
+]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = [
     "pyarrow.*",
 ]
 ignore_missing_imports = true
+follow_imports = "skip"
 
 [tool.pytest.ini_options]
 minversion = "8.0"
@@ -220,7 +235,7 @@ markers = [
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.7.0"
+version = "0.9.0"
 tag_format = "v$version"
 version_files = [
     "pyproject.toml:project.version",

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/src/loom/core/config/keys.py

@@ -20,6 +20,7 @@ class ConfigKey(StrEnum):
     DATABASE = "database"
     CELERY = "celery"
     KAFKA = "kafka"
+    MONGO = "mongo"
     STORAGE = "storage"
     STREAMING = "streaming"
     STREAMING_RUNTIME = "streaming.runtime"

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/src/loom/core/config/observability.py

@@ -2,10 +2,18 @@
 
 from __future__ import annotations
 
+import warnings
 from typing import Any
+from urllib.parse import urlparse
 
 from loom.core.model import LoomFrozenStruct
 
+_LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"}
+
+
+def _is_local_endpoint(endpoint: str) -> bool:
+    return (urlparse(endpoint).hostname or "") in _LOCAL_HOSTS
+
 
 class OtelConfig(LoomFrozenStruct, frozen=True):
     """OpenTelemetry SDK/exporter configuration.
@@ -29,7 +37,7 @@ class OtelConfig(LoomFrozenStruct, frozen=True):
     tracer_version: str = ""
     protocol: str = "http/protobuf"
     endpoint: str = ""
-    insecure: bool = True
+    insecure: bool = False
     headers: dict[str, str] = {}
     resource_attributes: dict[str, str] = {}
     span_attributes: dict[str, str] = {}
@@ -46,6 +54,13 @@ class OtelConfig(LoomFrozenStruct, frozen=True):
             raise ValueError(
                 "observability.otel_config.protocol must be either 'http/protobuf' or 'grpc'."
             )
+        if self.insecure and self.endpoint and not _is_local_endpoint(self.endpoint):
+            warnings.warn(
+                f"OtelConfig.insecure=True on non-local endpoint {self.endpoint!r}."
+                " Set insecure=False for production deployments.",
+                UserWarning,
+                stacklevel=2,
+            )
 
 
 __all__ = ["OtelConfig"]

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/src/loom/core/model/__init__.py

@@ -23,6 +23,7 @@ from loom.core.model.types import (
     DateTime,
     Float,
     Integer,
+    JsonStr,
     Numeric,
     String,
     Text,
@@ -45,6 +46,7 @@ __all__ = [
     "Float",
     "Integer",
     "JSON",
+    "JsonStr",
     "Postgres",
     "ProjectionField",
     "LoomStructMeta",

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/src/loom/core/model/types.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Annotated
+
 from loom.core.model.field import ColumnType
 
 
@@ -23,3 +25,12 @@ def DateTime(*, tz: bool = True) -> ColumnType:
 
 def Numeric(*, precision: int = 10, scale: int = 2) -> ColumnType:
     return ColumnType("Numeric", args=(precision, scale))
+
+
+class _JsonMarker:
+    """Sentinel that marks a str field as containing serialized JSON."""
+
+    __slots__ = ()
+
+
+JsonStr = Annotated[str, _JsonMarker()]

{loom_kernel-0.7.0 → loom_kernel-0.9.0}/src/loom/core/observability/config.py

@@ -13,10 +13,19 @@ class PrometheusConfig(LoomFrozenStruct, frozen=True, kw_only=True):
     """Prometheus scrape endpoint configuration.
 
     Args:
-        path: HTTP path where ``/metrics`` is served.
+        path: HTTP path where ``/metrics`` is served. Used by REST services
+            that mount the scrape endpoint on an existing FastAPI app.
+        port: Port for the standalone HTTP scrape server. Used by long-running
+            streaming processes that have no existing HTTP server.
+            ``None`` disables the standalone server (default).
+        bind_address: Network interface the standalone scrape server binds to.
+            Defaults to ``"127.0.0.1"`` (loopback only). Set to ``"0.0.0.0"``
+            only when the Prometheus scraper cannot reach the pod via loopback.
     """
 
     path: str = "/metrics"
+    port: int | None = None
+    bind_address: str = "127.0.0.1"
 
 
 class LogObservabilityConfig(LoomFrozenStruct, frozen=True, kw_only=True):