loom-kernel 2.17.2__tar.gz → 2.17.3__tar.gz

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/CHANGELOG.md

@@ -246,6 +246,11 @@
 
 ### etl
 
+- **etl:** `vacuum(full=True)` (`MaintainTable`, `MaintainSchema`,
+  `maintenance.vacuum.full`) also deletes files the Delta log does not
+  reference, such as the output of an aborted write, older than the retention
+  and honouring `dry_run`. It lists the whole table directory on each run.
+  Defaults to `False`.
 - **etl:** a step may receive values from the runner's config without reading
   the environment or taking them as params. `FromConfig("respondio.api_token")`
   (a `str`) or `FromConfig("respondio", RespondioSettings)` (a type
@@ -479,6 +484,15 @@
 
 ### etl
 
+- **etl:** `upsert` and `update` on the Polars backend write with the route's
+  `writer` and `commit` settings. The delta-rs MERGE received neither, so the
+  Parquet files it rewrote came out with delta-rs defaults (SNAPPY, row groups
+  of 1,048,576 rows) whatever the route declared, and its commit carried no
+  `custom_metadata`. The MERGE now takes the same `WriterProperties` and
+  `CommitProperties` as `append`, `replace`, `replace_partitions` and
+  `replace_where`. delta-rs MERGE takes no target file size: its files follow
+  the table property `delta.targetFileSize`, not the route's
+  `target_file_size`. Existing files keep their old settings until rewritten.
 - **etl:** historify collapses lazy runs without a `PerformanceWarning`.
 - **etl:** `IntoHistory(..., mode="snapshot")` no longer reopens a row on
   every run when a tracked column is null. Classification joined existing

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: loom-kernel
-Version: 2.17.2
+Version: 2.17.3
 Summary: Loom Python project
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/backends/polars/_writer.py

@@ -578,6 +578,7 @@ class PolarsTargetWriter(_WritePolicy[pl.LazyFrame, pl.DataFrame, PolarsPhysical
             predicate=merge_plan.predicate,
             source_alias=SOURCE_ALIAS,
             target_alias=TARGET_ALIAS,
+            **self._delta_properties(location),
         ).when_matched_update(updates=merge_plan.update_set)
         return merger, merge_plan, location
 
@@ -751,14 +752,21 @@ class PolarsTargetWriter(_WritePolicy[pl.LazyFrame, pl.DataFrame, PolarsPhysical
         )
 
     @staticmethod
-    def _write_kwargs(loc: TableLocation) -> dict[str, Any]:
+    def _delta_properties(loc: TableLocation) -> dict[str, Any]:
+        """Build ``writer_properties`` and ``commit_properties`` from TableLocation."""
+        return {
+            "writer_properties": WriterProperties(**loc.writer) if loc.writer else None,
+            "commit_properties": CommitProperties(**loc.commit) if loc.commit else None,
+        }
+
+    @classmethod
+    def _write_kwargs(cls, loc: TableLocation) -> dict[str, Any]:
         """Build write kwargs from TableLocation."""
         return {
             "storage_options": loc.storage_options or None,
             "configuration": loc.delta_config or None,
-            "writer_properties": WriterProperties(**loc.writer) if loc.writer else None,
             "target_file_size": loc.target_file_size,
-            "commit_properties": CommitProperties(**loc.commit) if loc.commit else None,
+            **cls._delta_properties(loc),
         }
 
     @staticmethod

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/maintenance/_builder.py

@@ -73,6 +73,7 @@ class MaintainTable:
         retention_hours: int | None = None,
         *,
         dry_run: bool = True,
+        full: bool = False,
     ) -> MaintainTable:
         """Schedule a VACUUM step.
 
@@ -82,8 +83,10 @@ class MaintainTable:
                 (default 168 h / 7 days).
             dry_run: When ``True`` (default), lists files without deleting.
                 Set ``dry_run=False`` explicitly for production runs.
+            full: When ``True``, also deletes files the Delta log does not
+                reference.  See :class:`~loom.etl.maintenance.VacuumSpec`.
         """
-        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run))
+        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run, full=full))
         return self
 
     def compact(self, target_size: int | None = None) -> MaintainTable:
@@ -162,9 +165,13 @@ class MaintainSchema:
         retention_hours: int | None = None,
         *,
         dry_run: bool = True,
+        full: bool = False,
     ) -> MaintainSchema:
-        """Schedule a VACUUM step for all discovered tables."""
-        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run))
+        """Schedule a VACUUM step for all discovered tables.
+
+        Takes the same arguments as :meth:`MaintainTable.vacuum`.
+        """
+        self._ops.append(VacuumSpec(retention_hours=retention_hours, dry_run=dry_run, full=full))
         return self
 
     def compact(self, target_size: int | None = None) -> MaintainSchema:

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/maintenance/_ops.py

@@ -31,11 +31,15 @@ class VacuumSpec:
             property (default 168 hours / 7 days).
         dry_run: When ``True`` (the default), lists eligible files without
             deleting them.  Set ``dry_run=False`` explicitly in production.
+        full: When ``True``, also deletes files the Delta log does not
+            reference, older than the retention.  ``False`` (the default)
+            deletes only the files the log records as removed.
     """
 
     name: ClassVar[str] = "vacuum"
     retention_hours: int | None = None
     dry_run: bool = True
+    full: bool = False
 
     def execute(
         self,

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/maintenance/_runner.py

@@ -171,7 +171,11 @@ class MaintenanceRunner:
             return MaintenanceReport()
 
         vacuum_spec = (
-            VacuumSpec(retention_hours=mc.vacuum.retention_hours, dry_run=mc.vacuum.dry_run)
+            VacuumSpec(
+                retention_hours=mc.vacuum.retention_hours,
+                dry_run=mc.vacuum.dry_run,
+                full=mc.vacuum.full,
+            )
             if mc.vacuum
             else None
         )

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/maintenance/backends/_delta_rs.py

@@ -40,9 +40,10 @@ class DeltaRsMaintainer:
         deleted = dt.vacuum(
             retention_hours=spec.retention_hours,
             dry_run=spec.dry_run,
+            full=spec.full,
             commit_properties=commit,
         )
-        _log.info("vacuum", uri=uri, dry_run=spec.dry_run, files=len(deleted))
+        _log.info("vacuum", uri=uri, dry_run=spec.dry_run, full=spec.full, files=len(deleted))
         return VacuumResult(files_deleted=deleted)
 
     def compact(

{loom_kernel-2.17.2 → loom_kernel-2.17.3}/src/loom/etl/storage/_config.py

@@ -307,10 +307,14 @@ class MaintenanceVacuumConfig(LoomFrozenStruct, frozen=True):
             (default 168 h / 7 days).
         dry_run: When ``True`` (default), lists files without deleting.
             Set ``dry_run=False`` explicitly for production runs.
+        full: When ``True``, also deletes files the Delta log does not
+            reference, older than the retention.  ``False`` (default) deletes
+            only the files the log records as removed.
     """
 
     retention_hours: int | None = None
     dry_run: bool = True
+    full: bool = False
 
 
 class MaintenanceConfig(LoomFrozenStruct, frozen=True):