loom-kernel 2.17.2__tar.gz → 2.18.0__tar.gz

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/CHANGELOG.md

@@ -179,6 +179,13 @@
 
 ### rest
 
+- **rest:** a `403` can say why and what is missing. `Forbidden` takes two
+  optional keyword arguments, `reason` (a machine-readable cause such as
+  `"mfa_required"`) and `missing` (the permissions the caller lacks, kept as a
+  tuple); `HttpErrorMapper` adds `reason` and `missing` to the body only when the
+  refusal carries them, so existing 403 bodies do not change. Both are public:
+  never put token claims or tenant data in them. An empty `reason`, a single
+  string as `missing` or an empty permission raise `ValueError`.
 - **rest:** a single `RuleViolation(field, message)` now answers its `field`.
   `HttpErrorMapper` dropped it, so two `rule_violation` 422s could only be
   told apart by their message text. The body adds `field` (`ErrorField.FIELD`,
@@ -246,6 +253,11 @@
 
 ### etl
 
+- **etl:** `vacuum(full=True)` (`MaintainTable`, `MaintainSchema`,
+  `maintenance.vacuum.full`) also deletes files the Delta log does not
+  reference, such as the output of an aborted write, older than the retention
+  and honouring `dry_run`. It lists the whole table directory on each run.
+  Defaults to `False`.
 - **etl:** a step may receive values from the runner's config without reading
   the environment or taking them as params. `FromConfig("respondio.api_token")`
   (a `str`) or `FromConfig("respondio", RespondioSettings)` (a type
@@ -479,6 +491,15 @@
 
 ### etl
 
+- **etl:** `upsert` and `update` on the Polars backend write with the route's
+  `writer` and `commit` settings. The delta-rs MERGE received neither, so the
+  Parquet files it rewrote came out with delta-rs defaults (SNAPPY, row groups
+  of 1,048,576 rows) whatever the route declared, and its commit carried no
+  `custom_metadata`. The MERGE now takes the same `WriterProperties` and
+  `CommitProperties` as `append`, `replace`, `replace_partitions` and
+  `replace_where`. delta-rs MERGE takes no target file size: its files follow
+  the table property `delta.targetFileSize`, not the route's
+  `target_file_size`. Existing files keep their old settings until rewritten.
 - **etl:** historify collapses lazy runs without a `PerformanceWarning`.
 - **etl:** `IntoHistory(..., mode="snapshot")` no longer reopens a row on
   every run when a tracked column is null. Classification joined existing

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: loom-kernel
-Version: 2.17.2
+Version: 2.18.0
 Summary: Loom Python project
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/core/errors/errors.py

@@ -97,10 +97,37 @@ class Forbidden(DomainError):
 
     Args:
         message: Description of the access restriction.
+        reason: Optional machine-readable cause a client can act on, such as
+            ``"mfa_required"``. ``None`` when the refusal has no specific cause.
+        missing: The permissions the caller lacks, in the order given; empty when
+            the authorizer does not tell them.
+
+    Both are keyword-only and public: the REST adapter puts them in the 403 body.
+    Never put token claims, tenant data or anything the caller may not learn in them;
+    leave them out where the refusal must not become an oracle for the policy.
+
+    Raises:
+        ValueError: If ``reason`` is empty, ``missing`` is a single string, or an item
+            of ``missing`` is not a non-empty string.
     """
 
-    def __init__(self, message: str = "Forbidden") -> None:
+    def __init__(
+        self,
+        message: str = "Forbidden",
+        *,
+        reason: str | None = None,
+        missing: Sequence[str] = (),
+    ) -> None:
+        if reason is not None and (not isinstance(reason, str) or not reason.strip()):
+            raise ValueError("Forbidden.reason must be a non-empty string or None")
+        if isinstance(missing, (str, bytes)):
+            raise ValueError("Forbidden.missing takes a sequence of permissions, not one string")
+        permissions = tuple(missing)
+        if any(not isinstance(item, str) or not item.strip() for item in permissions):
+            raise ValueError("Forbidden.missing items must be non-empty strings")
         super().__init__(message, code=ErrorCode.FORBIDDEN)
+        self.reason = reason
+        self.missing: tuple[str, ...] = permissions
 
 
 class Conflict(DomainError):

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/backends/polars/_writer.py

@@ -578,6 +578,7 @@ class PolarsTargetWriter(_WritePolicy[pl.LazyFrame, pl.DataFrame, PolarsPhysical
             predicate=merge_plan.predicate,
             source_alias=SOURCE_ALIAS,
             target_alias=TARGET_ALIAS,
+            **self._delta_properties(location),
         ).when_matched_update(updates=merge_plan.update_set)
         return merger, merge_plan, location
 
@@ -751,14 +752,21 @@ class PolarsTargetWriter(_WritePolicy[pl.LazyFrame, pl.DataFrame, PolarsPhysical
         )
 
     @staticmethod
-    def _write_kwargs(loc: TableLocation) -> dict[str, Any]:
+    def _delta_properties(loc: TableLocation) -> dict[str, Any]:
+        """Build ``writer_properties`` and ``commit_properties`` from TableLocation."""
+        return {
+            "writer_properties": WriterProperties(**loc.writer) if loc.writer else None,
+            "commit_properties": CommitProperties(**loc.commit) if loc.commit else None,
+        }
+
+    @classmethod
+    def _write_kwargs(cls, loc: TableLocation) -> dict[str, Any]:
         """Build write kwargs from TableLocation."""
         return {
             "storage_options": loc.storage_options or None,
             "configuration": loc.delta_config or None,
-            "writer_properties": WriterProperties(**loc.writer) if loc.writer else None,
             "target_file_size": loc.target_file_size,
-            "commit_properties": CommitProperties(**loc.commit) if loc.commit else None,
+            **cls._delta_properties(loc),
         }
 
     @staticmethod

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/maintenance/_builder.py

@@ -73,6 +73,7 @@ class MaintainTable:
         retention_hours: int | None = None,
         *,
         dry_run: bool = True,
+        full: bool = False,
     ) -> MaintainTable:
         """Schedule a VACUUM step.
 
@@ -82,8 +83,10 @@ class MaintainTable:
                 (default 168 h / 7 days).
             dry_run: When ``True`` (default), lists files without deleting.
                 Set ``dry_run=False`` explicitly for production runs.
+            full: When ``True``, also deletes files the Delta log does not
+                reference.  See :class:`~loom.etl.maintenance.VacuumSpec`.
         """
-        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run))
+        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run, full=full))
         return self
 
     def compact(self, target_size: int | None = None) -> MaintainTable:
@@ -162,9 +165,13 @@ class MaintainSchema:
         retention_hours: int | None = None,
         *,
         dry_run: bool = True,
+        full: bool = False,
     ) -> MaintainSchema:
-        """Schedule a VACUUM step for all discovered tables."""
-        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run))
+        """Schedule a VACUUM step for all discovered tables.
+
+        Takes the same arguments as :meth:`MaintainTable.vacuum`.
+        """
+        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run, full=full))
         return self
 
     def compact(self, target_size: int | None = None) -> MaintainSchema:

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/maintenance/_ops.py

@@ -31,11 +31,15 @@ class VacuumSpec:
             property (default 168 hours / 7 days).
         dry_run: When ``True`` (the default), lists eligible files without
             deleting them.  Set ``dry_run=False`` explicitly in production.
+        full: When ``True``, also deletes files the Delta log does not
+            reference, older than the retention.  ``False`` (the default)
+            deletes only the files the log records as removed.
     """
 
     name: ClassVar[str] = "vacuum"
     retention_hours: int | None = None
     dry_run: bool = True
+    full: bool = False
 
     def execute(
         self,

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/maintenance/_runner.py

@@ -171,7 +171,11 @@ class MaintenanceRunner:
             return MaintenanceReport()
 
         vacuum_spec = (
-            VacuumSpec(retention_hours=mc.vacuum.retention_hours, dry_run=mc.vacuum.dry_run)
+            VacuumSpec(
+                retention_hours=mc.vacuum.retention_hours,
+                dry_run=mc.vacuum.dry_run,
+                full=mc.vacuum.full,
+            )
             if mc.vacuum
             else None
         )

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/maintenance/backends/_delta_rs.py

@@ -40,9 +40,10 @@ class DeltaRsMaintainer:
         deleted = dt.vacuum(
             retention_hours=spec.retention_hours,
             dry_run=spec.dry_run,
+            full=spec.full,
             commit_properties=commit,
         )
-        _log.info("vacuum", uri=uri, dry_run=spec.dry_run, files=len(deleted))
+        _log.info("vacuum", uri=uri, dry_run=spec.dry_run, full=spec.full, files=len(deleted))
         return VacuumResult(files_deleted=deleted)
 
     def compact(

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/etl/storage/_config.py

@@ -307,10 +307,14 @@ class MaintenanceVacuumConfig(LoomFrozenStruct, frozen=True):
             (default 168 h / 7 days).
         dry_run: When ``True`` (default), lists files without deleting.
             Set ``dry_run=False`` explicitly for production runs.
+        full: When ``True``, also deletes files the Delta log does not
+            reference, older than the retention.  ``False`` (default) deletes
+            only the files the log records as removed.
     """
 
     retention_hours: int | None = None
     dry_run: bool = True
+    full: bool = False
 
 
 class MaintenanceConfig(LoomFrozenStruct, frozen=True):

{loom_kernel-2.17.2 → loom_kernel-2.18.0}/src/loom/rest/errors.py

@@ -9,7 +9,7 @@ from typing import Any, ClassVar
 from fastapi import HTTPException
 
 from loom.core.engine.post_commit import PostCommitError
-from loom.core.errors import LoomError, NotFound, RuleViolation, RuleViolations
+from loom.core.errors import Forbidden, LoomError, NotFound, RuleViolation, RuleViolations
 from loom.core.errors.codes import ErrorCode
 from loom.core.model import BoundaryValidationError
 from loom.core.tracing import get_trace_id
@@ -41,6 +41,8 @@ class ErrorField(StrEnum):
     VIOLATIONS = "violations"
     FIELD = "field"
     COMMITTED = "committed"
+    REASON = "reason"
+    MISSING = "missing"
 
 
 class HttpErrorMapper:
@@ -52,6 +54,8 @@ class HttpErrorMapper:
     per error type:
 
     - :class:`~loom.core.errors.NotFound` → ``entity``, ``id``
+    - :class:`~loom.core.errors.Forbidden` → ``reason`` and ``missing``, each only
+      when the refusal carries it
     - :class:`~loom.core.errors.RuleViolation` → ``field``
     - :class:`~loom.core.errors.RuleViolations` → ``violations``
     - :class:`~loom.core.model.BoundaryValidationError` → ``violations``
@@ -109,6 +113,12 @@ class HttpErrorMapper:
             detail[ErrorField.ENTITY] = error.entity
             detail[ErrorField.ID] = error.id
 
+        if isinstance(error, Forbidden):
+            if error.reason is not None:
+                detail[ErrorField.REASON] = error.reason
+            if error.missing:
+                detail[ErrorField.MISSING] = list(error.missing)
+
         if isinstance(error, RuleViolation):
             detail[ErrorField.FIELD] = error.field