databricks-bundle-decorators 0.12.2__tar.gz → 0.12.4__tar.gz

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: databricks-bundle-decorators
-Version: 0.12.2
+Version: 0.12.4
 Summary: Decorator-based framework for defining Databricks jobs and tasks as Python code.
 Author: boccileonardo
 Author-email: boccileonardo <leonardobocci99@hotmail.com>

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/pyproject.toml

@@ -4,7 +4,7 @@ requires = [ "uv-build>=0.10.4,<1" ]
 
 [project]
 name = "databricks-bundle-decorators"
-version = "0.12.2"
+version = "0.12.4"
 description = "Decorator-based framework for defining Databricks jobs and tasks as Python code."
 readme = "README.md"
 authors = [ { name = "boccileonardo", email = "leonardobocci99@hotmail.com" } ]

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/backfill.py

@@ -278,6 +278,27 @@ class DailyBackfill(BackfillDef):
             if self.data_lag:
                 e = e.subtract(days=self.data_lag)
 
+        # Clamp overrides to definition bounds
+        defn_start = self._parse(self.start_date)
+        if s < defn_start:
+            _logger.warning(
+                "Start '%s' is before the backfill start_date '%s'; "
+                "skipping out-of-range periods",
+                start,
+                self.start_date,
+            )
+            s = defn_start
+        if self.end_date is not None:
+            defn_end = self._parse(self.end_date)
+            if e > defn_end:
+                _logger.warning(
+                    "End '%s' is after the backfill end_date '%s'; "
+                    "skipping out-of-range periods",
+                    end,
+                    self.end_date,
+                )
+                e = defn_end
+
         keys: list[str] = []
         while s <= e:
             keys.append(s.format(self._FMT))
@@ -354,6 +375,27 @@ class WeeklyBackfill(BackfillDef):
             weekday_offset = today.day_of_week().value - 1  # 0 for Monday
             e = today.subtract(days=weekday_offset)
 
+        # Clamp overrides to definition bounds
+        defn_start = self._parse_iso_week(self.start_date)
+        if s < defn_start:
+            _logger.warning(
+                "Start '%s' is before the backfill start_date '%s'; "
+                "skipping out-of-range periods",
+                start,
+                self.start_date,
+            )
+            s = defn_start
+        if self.end_date is not None:
+            defn_end = self._parse_iso_week(self.end_date)
+            if e > defn_end:
+                _logger.warning(
+                    "End '%s' is after the backfill end_date '%s'; "
+                    "skipping out-of-range periods",
+                    end,
+                    self.end_date,
+                )
+                e = defn_end
+
         keys: list[str] = []
         while s <= e:
             keys.append(self._fmt_iso_week(s))
@@ -422,6 +464,27 @@ class MonthlyBackfill(BackfillDef):
             # Current month
             e = today.replace(day=1)
 
+        # Clamp overrides to definition bounds
+        defn_start = self._parse_month(self.start_date)
+        if s < defn_start:
+            _logger.warning(
+                "Start '%s' is before the backfill start_date '%s'; "
+                "skipping out-of-range periods",
+                start,
+                self.start_date,
+            )
+            s = defn_start
+        if self.end_date is not None:
+            defn_end = self._parse_month(self.end_date)
+            if e > defn_end:
+                _logger.warning(
+                    "End '%s' is after the backfill end_date '%s'; "
+                    "skipping out-of-range periods",
+                    end,
+                    self.end_date,
+                )
+                e = defn_end
+
         keys: list[str] = []
         while s <= e:
             keys.append(s.format(self._FMT))
@@ -502,6 +565,27 @@ class HourlyBackfill(BackfillDef):
             if self.data_lag:
                 e = e.subtract(hours=self.data_lag)
 
+        # Clamp overrides to definition bounds
+        defn_start = self._parse_hour(self.start_date)
+        if s < defn_start:
+            _logger.warning(
+                "Start '%s' is before the backfill start_date '%s'; "
+                "skipping out-of-range periods",
+                start,
+                self.start_date,
+            )
+            s = defn_start
+        if self.end_date is not None:
+            defn_end = self._parse_hour(self.end_date)
+            if e > defn_end:
+                _logger.warning(
+                    "End '%s' is after the backfill end_date '%s'; "
+                    "skipping out-of-range periods",
+                    end,
+                    self.end_date,
+                )
+                e = defn_end
+
         keys: list[str] = []
         seen: set[str] = set()
         cur = s

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/io_manager.py

@@ -354,13 +354,13 @@ class IoManager(ABC):
             before_sleep_log,
             retry,
             stop_after_attempt,
-            wait_exponential,
+            wait_exponential_jitter,
         )
 
         retryer = retry(
             stop=stop_after_attempt(self.retry.max_attempts),
-            wait=wait_exponential(
-                multiplier=self.retry.delay,
+            wait=wait_exponential_jitter(
+                initial=self.retry.delay,
                 exp_base=self.retry.backoff_factor,
             ),
             reraise=True,

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/io_managers/polars_delta.py

@@ -202,7 +202,12 @@ class PolarsDeltaIoManager(IoManager):
             merger = obj._build_merger(uri, storage_options=self.storage_options)
             if merger is None:
                 # Target table doesn't exist yet — write source data directly.
-                obj._initial_write(uri, storage_options=self.storage_options)
+                obj._initial_write(
+                    uri,
+                    storage_options=self.storage_options,
+                    partition_by=context.partition_by,
+                    write_options=dict(self._write_options),
+                )
             else:
                 merger.execute()
             self._last_partition_values = {}
@@ -287,7 +292,12 @@ class PolarsDeltaIoManager(IoManager):
             def _execute_merge() -> None:
                 merger = obj._build_merger(uri, storage_options=self.storage_options)
                 if merger is None:
-                    obj._initial_write(uri, storage_options=self.storage_options)
+                    obj._initial_write(
+                        uri,
+                        storage_options=self.storage_options,
+                        partition_by=context.partition_by,
+                        write_options=dict(self._write_options),
+                    )
                 else:
                     merger.execute()
 

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/io_managers/spark_delta.py

@@ -89,7 +89,11 @@ class _SparkDeltaBase(IoManager):
             )
             builder = obj._build_spark_merger(uri)
             if builder is None:
-                obj._initial_spark_write(uri)
+                obj._initial_spark_write(
+                    uri,
+                    partition_by=context.partition_by,
+                    write_options=dict(self._write_options),
+                )
             else:
                 builder.execute()
             self._last_partition_values = {}

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/io_managers/spark_uc.py

@@ -147,7 +147,11 @@ class SparkUCTableIoManager(IoManager):
             )
             builder = obj._build_spark_merger(table)
             if builder is None:
-                obj._initial_spark_write(table)
+                obj._initial_spark_write(
+                    table,
+                    partition_by=context.partition_by,
+                    write_options=dict(self._write_options),
+                )
             else:
                 builder.execute()
             self._last_partition_values = {}
@@ -325,7 +329,11 @@ class SparkUCVolumeDeltaIoManager(IoManager):
             )
             builder = obj._build_spark_merger(uri)
             if builder is None:
-                obj._initial_spark_write(uri)
+                obj._initial_spark_write(
+                    uri,
+                    partition_by=context.partition_by,
+                    write_options=dict(self._write_options),
+                )
             else:
                 builder.execute()
             self._last_partition_values = {}

{databricks_bundle_decorators-0.12.2 → databricks_bundle_decorators-0.12.4}/src/databricks_bundle_decorators/merge.py

@@ -256,7 +256,11 @@ class DeltaMerge:
         return type(self.source).__module__.startswith("pyspark.")
 
     def _initial_write(
-        self, table_uri: str, storage_options: dict[str, str] | None = None
+        self,
+        table_uri: str,
+        storage_options: dict[str, str] | None = None,
+        partition_by: list[str] | None = None,
+        write_options: dict[str, Any] | None = None,
     ) -> None:
         """Write source data directly when the target table doesn't exist yet.
 
@@ -265,15 +269,20 @@ class DeltaMerge:
         """
         import polars as pl  # noqa: PLC0415
 
+        opts: dict[str, Any] = dict(write_options or {})
+        if partition_by:
+            delta_opts: dict[str, Any] = opts.setdefault("delta_write_options", {})
+            delta_opts.setdefault("partition_by", partition_by)
+
         source = self.source
         if isinstance(source, pl.LazyFrame):
-            source.sink_delta(table_uri, storage_options=storage_options)
+            source.sink_delta(table_uri, storage_options=storage_options, **opts)
         elif isinstance(source, pl.DataFrame):
-            source.write_delta(table_uri, storage_options=storage_options)
+            source.write_delta(table_uri, storage_options=storage_options, **opts)
         else:
             # PyArrow or other — convert to polars first
             df = pl.DataFrame(pl.from_arrow(source))
-            df.write_delta(table_uri, storage_options=storage_options)
+            df.write_delta(table_uri, storage_options=storage_options, **opts)
 
     def _build_spark_merger(self, table_identifier: str) -> Any:
         """Build a fresh ``delta.tables.DeltaMergeBuilder`` for Spark.
@@ -329,7 +338,12 @@ class DeltaMerge:
 
         return builder
 
-    def _initial_spark_write(self, table_identifier: str) -> None:
+    def _initial_spark_write(
+        self,
+        table_identifier: str,
+        partition_by: list[str] | None = None,
+        write_options: dict[str, str] | None = None,
+    ) -> None:
         """Write source Spark DataFrame directly when the target doesn't exist.
 
         Called by the Spark IoManager on first run.
@@ -339,6 +353,10 @@ class DeltaMerge:
         )
 
         writer = self.source.write.format("delta").mode("error")
+        if partition_by:
+            writer = writer.partitionBy(*partition_by)
+        for k, v in (write_options or {}).items():
+            writer = writer.option(k, v)
         if is_path:
             writer.save(table_identifier)
         else: