databricks4py 0.2.0__tar.gz → 0.3.0__tar.gz

{databricks4py-0.2.0/src/databricks4py.egg-info → databricks4py-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks4py
-Version: 0.2.0
+Version: 0.3.0
 Summary: Spark, Delta Lake, and Databricks utility library for Python
 Author: kirankbs
 License: MIT

{databricks4py-0.2.0 → databricks4py-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "databricks4py"
-version = "0.2.0"
+version = "0.3.0"
 description = "Spark, Delta Lake, and Databricks utility library for Python"
 readme = "README.md"
 license = {text = "MIT"}

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/__init__.py

@@ -4,7 +4,7 @@ A collection of reusable abstractions for building PySpark applications
 on Databricks and locally.
 """
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 from databricks4py.catalog import CatalogSchema
 from databricks4py.config import Environment, JobConfig, UnityConfig

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/io/__init__.py

@@ -10,8 +10,13 @@ from databricks4py.io.delta import (
     optimize_table,
     vacuum_table,
 )
+from databricks4py.io.maintenance import MaintenanceResult, MaintenanceRunner, analyze_table
 from databricks4py.io.merge import MergeBuilder, MergeResult
-from databricks4py.io.streaming import StreamingTableReader, StreamingTriggerOptions
+from databricks4py.io.streaming import (
+    CircuitBreakerError,
+    StreamingTableReader,
+    StreamingTriggerOptions,
+)
 
 __all__ = [
     # Checkpoint
@@ -31,10 +36,15 @@ __all__ = [
     "GeneratedColumn",
     "optimize_table",
     "vacuum_table",
+    # Maintenance
+    "MaintenanceResult",
+    "MaintenanceRunner",
+    "analyze_table",
     # Merge
     "MergeBuilder",
     "MergeResult",
     # Streaming
+    "CircuitBreakerError",
     "StreamingTableReader",
     "StreamingTriggerOptions",
 ]

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/io/delta.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import logging
 from collections.abc import Sequence
 from dataclasses import dataclass
+from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
 from pyspark.sql import DataFrame, SparkSession
@@ -95,7 +96,7 @@ class DeltaTable:
         spark: SparkSession | None = None,
     ) -> None:
         self._spark = active_fallback(spark)
-        self._table_name = table_name
+        self._table_name = self._validated_table_name(table_name)
         self._schema = self._resolve_schema(schema)
         self._location = location
         self._generated_columns = list(generated_columns or [])
@@ -107,6 +108,20 @@ class DeltaTable:
 
         self._ensure_table_exists()
 
+    @staticmethod
+    def _validated_table_name(name: str) -> str:
+        """Validate that a table name doesn't contain SQL injection characters.
+
+        Allows word characters, dots, backticks, hyphens, spaces, and quotes
+        (all valid in Spark/Delta table identifiers). Rejects semicolons and
+        SQL comment sequences that are the primary injection vectors.
+        """
+        import re
+
+        if re.search(r"--|;|/\*|\*/", name):
+            raise ValueError(f"Unsafe characters in table name: {name!r}")
+        return name
+
     @staticmethod
     def _resolve_schema(schema: StructType | dict[str, str]) -> StructType:
         if isinstance(schema, dict):
@@ -251,6 +266,46 @@ class DeltaTable:
         row = self.detail().select("partitionColumns").first()
         return list(row["partitionColumns"]) if row else []
 
+    def history(self, limit: int = 20) -> DataFrame:
+        """Retrieve the Delta transaction history for this table.
+
+        Args:
+            limit: Maximum number of history entries to return (default 20).
+
+        Returns:
+            DataFrame with columns including ``version``, ``timestamp``,
+            ``operation``, and ``operationMetrics``.
+        """
+        return self._spark.sql(f"DESCRIBE HISTORY {self._table_name} LIMIT {limit}")
+
+    def restore(self, version: int) -> DataFrame:
+        """Restore this table to a prior Delta version.
+
+        Args:
+            version: The Delta version number to restore to (from :meth:`history`).
+
+        Returns:
+            DataFrame summarising the restore operation.
+        """
+        logger.info("Restoring %s to version %d", self._table_name, version)
+        return self._spark.sql(f"RESTORE TABLE {self._table_name} TO VERSION AS OF {version}")
+
+    def restore_to_timestamp(self, timestamp: datetime) -> DataFrame:
+        """Restore this table to the state at a given timestamp.
+
+        The ``timestamp`` is formatted as ``YYYY-MM-DD HH:MM:SS`` for the SQL
+        statement. Pass a timezone-aware datetime for predictable behaviour.
+
+        Args:
+            timestamp: The point in time to restore to.
+
+        Returns:
+            DataFrame summarising the restore operation.
+        """
+        ts_str = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+        logger.info("Restoring %s to timestamp %s", self._table_name, ts_str)
+        return self._spark.sql(f"RESTORE TABLE {self._table_name} TO TIMESTAMP AS OF '{ts_str}'")
+
     def merge(
         self,
         source: DataFrame,
@@ -359,7 +414,7 @@ class DeltaTable:
         history = self._spark.sql(f"DESCRIBE HISTORY {self._table_name} LIMIT 1")
         rows = history.collect()
         if not rows:
-            return
+            return _MergeResult(rows_inserted=0, rows_updated=0, rows_deleted=0)
         metrics: dict[str, str] = rows[0]["operationMetrics"] or {}
 
         result = _MergeResult(

databricks4py-0.3.0/src/databricks4py/io/maintenance.py

@@ -0,0 +1,151 @@
+"""Table maintenance utilities for Delta Lake."""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyspark.sql import SparkSession
+
+from databricks4py.io.delta import optimize_table, vacuum_table
+from databricks4py.spark_session import active_fallback
+
+if TYPE_CHECKING:
+    from databricks4py.metrics.base import MetricsSink
+
+__all__ = [
+    "MaintenanceResult",
+    "MaintenanceRunner",
+    "analyze_table",
+]
+
+logger = logging.getLogger(__name__)
+
+
+def analyze_table(
+    table_name: str,
+    *,
+    columns: Sequence[str] | None = None,
+    spark: SparkSession | None = None,
+) -> None:
+    """Run ANALYZE TABLE to compute statistics.
+
+    Args:
+        table_name: The table to analyze.
+        columns: Optional columns to compute statistics for.
+        spark: Optional SparkSession.
+    """
+    _spark = active_fallback(spark)
+
+    if columns:
+        col_list = ", ".join(columns)
+        sql = f"ANALYZE TABLE {table_name} COMPUTE STATISTICS FOR COLUMNS {col_list}"
+    else:
+        sql = f"ANALYZE TABLE {table_name} COMPUTE STATISTICS"
+
+    logger.info("Running: %s", sql)
+    _spark.sql(sql)
+
+
+@dataclass(frozen=True)
+class MaintenanceResult:
+    """Result of a maintenance run."""
+
+    table_name: str
+    optimized: bool
+    vacuumed: bool
+    analyzed: bool
+    duration_ms: float
+
+
+class MaintenanceRunner:
+    """Composes optimize, vacuum, and analyze into a single maintenance run."""
+
+    def __init__(
+        self,
+        table_name: str,
+        *,
+        optimize: bool = True,
+        vacuum: bool = True,
+        analyze: bool = True,
+        zorder_by: str | Sequence[str] | None = None,
+        retention_hours: int = 168,
+        analyze_columns: Sequence[str] | None = None,
+        metrics_sink: MetricsSink | None = None,
+        spark: SparkSession | None = None,
+    ) -> None:
+        self._table_name = table_name
+        self._optimize = optimize
+        self._vacuum = vacuum
+        self._analyze = analyze
+        self._zorder_by = zorder_by
+        self._retention_hours = retention_hours
+        self._analyze_columns = analyze_columns
+        self._metrics_sink = metrics_sink
+        self._spark = active_fallback(spark)
+
+    def run(self) -> MaintenanceResult:
+        start = time.monotonic()
+
+        if self._optimize:
+            logger.info("Optimizing %s", self._table_name)
+            optimize_table(
+                self._table_name,
+                zorder_by=self._zorder_by,
+                spark=self._spark,
+            )
+
+        if self._vacuum:
+            logger.info("Vacuuming %s", self._table_name)
+            vacuum_table(
+                self._table_name,
+                retention_hours=self._retention_hours,
+                spark=self._spark,
+            )
+
+        if self._analyze:
+            logger.info("Analyzing %s", self._table_name)
+            analyze_table(
+                self._table_name,
+                columns=self._analyze_columns,
+                spark=self._spark,
+            )
+
+        duration_ms = (time.monotonic() - start) * 1000
+
+        result = MaintenanceResult(
+            table_name=self._table_name,
+            optimized=self._optimize,
+            vacuumed=self._vacuum,
+            analyzed=self._analyze,
+            duration_ms=duration_ms,
+        )
+
+        if self._metrics_sink:
+            from datetime import datetime, timezone
+
+            from databricks4py.metrics.base import MetricEvent
+
+            event = MetricEvent(
+                job_name="maintenance",
+                event_type="maintenance_complete",
+                timestamp=datetime.now(tz=timezone.utc),
+                duration_ms=int(duration_ms),
+                table_name=self._table_name,
+                metadata={
+                    "optimized": result.optimized,
+                    "vacuumed": result.vacuumed,
+                    "analyzed": result.analyzed,
+                },
+            )
+            self._metrics_sink.emit(event)
+
+        logger.info(
+            "Maintenance complete for %s in %.0fms",
+            self._table_name,
+            duration_ms,
+        )
+        return result

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/io/merge.py

@@ -92,6 +92,45 @@ class MergeBuilder:
         self._actions.append({"type": "not_matched_by_source_delete", "condition": condition})
         return self
 
+    def when_matched_soft_delete(
+        self,
+        *,
+        deleted_col: str = "is_deleted",
+        deleted_at_col: str | None = "deleted_at",
+        condition: str | None = None,
+    ) -> MergeBuilder:
+        """Mark matched rows as logically deleted without physically removing them.
+
+        Sets ``deleted_col`` to ``True`` and optionally ``deleted_at_col`` to
+        ``current_timestamp()``. Useful for audit-friendly delete patterns in
+        regulated environments.
+
+        Args:
+            deleted_col: Boolean column to set ``True`` on the matched row.
+            deleted_at_col: Timestamp column to set to ``current_timestamp()``.
+                Pass ``None`` to skip updating a timestamp column.
+            condition: Optional SQL condition restricting which matched rows to affect.
+
+        Returns:
+            This builder for chaining.
+
+        Raises:
+            ValueError: If ``deleted_col`` or ``deleted_at_col`` is an empty string.
+        """
+        if not deleted_col:
+            raise ValueError("deleted_col must be a non-empty column name")
+        if deleted_at_col is not None and not deleted_at_col:
+            raise ValueError("deleted_at_col must be a non-empty column name or None")
+        self._actions.append(
+            {
+                "type": "matched_soft_delete",
+                "deleted_col": deleted_col,
+                "deleted_at_col": deleted_at_col,
+                "condition": condition,
+            }
+        )
+        return self
+
     def _build_condition(self) -> str:
         if self._join_condition:
             return self._join_condition
@@ -144,6 +183,19 @@ class MergeBuilder:
                 return merger.whenNotMatchedBySourceDelete(condition=cond)
             return merger.whenNotMatchedBySourceDelete()
 
+        if action_type == "matched_soft_delete":
+            deleted_col = action["deleted_col"]
+            deleted_at_col = action.get("deleted_at_col")
+            cond = action.get("condition")
+
+            update_map: dict[str, str] = {deleted_col: "true"}
+            if deleted_at_col:
+                update_map[deleted_at_col] = "current_timestamp()"
+
+            if cond:
+                return merger.whenMatchedUpdate(condition=cond, set=update_map)
+            return merger.whenMatchedUpdate(set=update_map)
+
         msg = f"Unknown merge action: {action_type}"
         raise ValueError(msg)
 

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/io/streaming.py

@@ -19,6 +19,7 @@ if TYPE_CHECKING:
     from databricks4py.metrics.base import MetricsSink
 
 __all__ = [
+    "CircuitBreakerError",
     "StreamingTableReader",
     "StreamingTriggerOptions",
 ]
@@ -26,6 +27,14 @@ __all__ = [
 logger = logging.getLogger(__name__)
 
 
+class CircuitBreakerError(RuntimeError):
+    """Raised when the streaming circuit breaker trips due to consecutive failures.
+
+    Stops the query so the issue can be investigated rather than silently
+    accumulating data in the DLQ.
+    """
+
+
 class StreamingTriggerOptions(Enum):
     """Common streaming trigger configurations.
 
@@ -83,6 +92,10 @@ class StreamingTableReader(ABC):
             is written here with ``_dlq_error_message``, ``_dlq_error_timestamp``,
             and ``_dlq_batch_id`` columns appended. Uses ``mergeSchema=true`` so
             the table is auto-created on first failure.
+        max_consecutive_failures: If set, raises CircuitBreakerError on the Nth
+            consecutive batch failure (i.e. ``max_consecutive_failures=1`` means the
+            very first failure trips the breaker). Applies whether or not a DLQ is
+            configured. Resets to zero after the first subsequent successful batch.
         spark: Optional SparkSession.
     """
 
@@ -101,12 +114,15 @@ class StreamingTableReader(ABC):
         checkpoint_manager: CheckpointManager | None = None,
         metrics_sink: MetricsSink | None = None,
         dead_letter_table: str | None = None,
+        max_consecutive_failures: int | None = None,
         spark: SparkSession | None = None,
     ) -> None:
         self._spark = active_fallback(spark)
         self._source_table = source_table
         self._metrics_sink = metrics_sink
         self._dead_letter_table = dead_letter_table
+        self._max_consecutive_failures = max_consecutive_failures
+        self._consecutive_failures: int = 0
         self._query: StreamingQuery | None = None
 
         # Resolve trigger to a plain dict
@@ -169,6 +185,17 @@ class StreamingTableReader(ABC):
             error_msg[:200],
         )
 
+    def _check_circuit_breaker(self, cause: BaseException) -> None:
+        """Raise CircuitBreakerError if the failure threshold has been exceeded."""
+        if (
+            self._max_consecutive_failures is not None
+            and self._consecutive_failures >= self._max_consecutive_failures
+        ):
+            raise CircuitBreakerError(
+                f"Circuit breaker tripped after {self._consecutive_failures} consecutive "
+                f"failures on {self._source_table}"
+            ) from cause
+
     def _foreach_batch_wrapper(self, df: DataFrame, batch_id: int) -> None:
         """Internal wrapper handling empty batch detection, filtering, and metrics."""
         if self._skip_empty_batches and df.isEmpty():
@@ -187,7 +214,9 @@ class StreamingTableReader(ABC):
         start = time.monotonic()
         try:
             self.process_batch(df, batch_id)
-        except Exception:  # noqa: BLE001 — broad catch intentional: DLQ must capture all failures
+        except Exception as exc:  # noqa: BLE001 — broad catch intentional: DLQ must capture all failures
+            self._consecutive_failures += 1
+
             if self._dead_letter_table is not None:
                 import traceback
 
@@ -202,8 +231,13 @@ class StreamingTableReader(ABC):
                         exc_info=True,
                     )
                     raise
+                self._check_circuit_breaker(exc)
                 return
+
+            self._check_circuit_breaker(exc)
             raise
+        else:
+            self._consecutive_failures = 0
         duration_ms = (time.monotonic() - start) * 1000
 
         if self._metrics_sink is not None:

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/quality/__init__.py

@@ -3,6 +3,7 @@
 from databricks4py.quality.base import Expectation, ExpectationResult, QualityReport
 from databricks4py.quality.expectations import (
     ColumnExists,
+    FreshnessExpectation,
     InRange,
     MatchesRegex,
     NotNull,
@@ -10,11 +11,13 @@ from databricks4py.quality.expectations import (
     Unique,
 )
 from databricks4py.quality.gate import QualityError, QualityGate
+from databricks4py.quality.profiler import ColumnProfile, DataProfile, profile
 
 __all__ = [
     "ColumnExists",
     "Expectation",
     "ExpectationResult",
+    "FreshnessExpectation",
     "InRange",
     "MatchesRegex",
     "NotNull",
@@ -23,4 +26,8 @@ __all__ = [
     "QualityReport",
     "RowCount",
     "Unique",
+    # Profiler
+    "ColumnProfile",
+    "DataProfile",
+    "profile",
 ]

{databricks4py-0.2.0 → databricks4py-0.3.0}/src/databricks4py/quality/expectations.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from datetime import timedelta
 from functools import reduce
 from typing import TYPE_CHECKING
 
@@ -10,7 +11,15 @@ from databricks4py.quality.base import Expectation, ExpectationResult
 if TYPE_CHECKING:
     from pyspark.sql import Column, DataFrame
 
-__all__ = ["ColumnExists", "InRange", "MatchesRegex", "NotNull", "RowCount", "Unique"]
+__all__ = [
+    "ColumnExists",
+    "FreshnessExpectation",
+    "InRange",
+    "MatchesRegex",
+    "NotNull",
+    "RowCount",
+    "Unique",
+]
 
 
 class NotNull(Expectation):
@@ -182,3 +191,83 @@ class ColumnExists(Expectation):
         if self._dtype:
             return f"ColumnExists({cols}, dtype={self._dtype!r})"
         return f"ColumnExists({cols})"
+
+
+class FreshnessExpectation(Expectation):
+    """Validates that a timestamp column contains sufficiently recent data.
+
+    The table passes when ``max(column) >= now - max_age``. The failing condition
+    identifies individual rows older than the cutoff, which is useful for quarantine
+    routing via :class:`~databricks4py.quality.gate.QualityGate`.
+
+    Args:
+        column: Timestamp column to check.
+        max_age: Maximum allowed age of the most-recent row. A ``timedelta(hours=1)``
+            means the newest row must be within the last hour.
+
+    Example::
+
+        from datetime import timedelta
+        from databricks4py.quality.expectations import FreshnessExpectation
+
+        expectation = FreshnessExpectation("event_time", max_age=timedelta(hours=6))
+        result = expectation.validate(df)
+        if not result.passed:
+            print(result)
+    """
+
+    def __init__(self, column: str, max_age: timedelta) -> None:
+        self._column = column
+        self._max_age = max_age
+
+    def validate(self, df: DataFrame) -> ExpectationResult:
+        from datetime import datetime, timezone
+
+        from pyspark.sql import functions as F
+
+        total = df.count()
+        if total == 0:
+            return ExpectationResult(
+                expectation=repr(self),
+                passed=False,
+                total_rows=0,
+                failing_rows=0,
+            )
+
+        cutoff = datetime.now(tz=timezone.utc) - self._max_age
+
+        max_row = df.agg(F.max(F.col(self._column)).alias("max_ts")).first()
+        max_ts = max_row["max_ts"] if max_row else None
+
+        if max_ts is None:
+            return ExpectationResult(
+                expectation=repr(self),
+                passed=False,
+                total_rows=total,
+                failing_rows=0,
+            )
+
+        max_ts_utc = max_ts.replace(tzinfo=timezone.utc) if max_ts.tzinfo is None else max_ts
+        passed = max_ts_utc >= cutoff
+
+        # Use the same cutoff computed above so row-count and table-level verdict are consistent
+        stale_condition = F.col(self._column) < F.lit(cutoff)
+        failing = df.where(stale_condition).count()
+
+        return ExpectationResult(
+            expectation=repr(self),
+            passed=passed,
+            total_rows=total,
+            failing_rows=failing,
+        )
+
+    def failing_condition(self) -> Column | None:
+        from datetime import datetime, timezone
+
+        from pyspark.sql import functions as F
+
+        cutoff = datetime.now(tz=timezone.utc) - self._max_age
+        return F.col(self._column) < F.lit(cutoff)
+
+    def __repr__(self) -> str:
+        return f"FreshnessExpectation({self._column!r}, max_age={self._max_age!r})"