delta-engine 0.1.0__py3-none-any.whl

delta_engine/__init__.py

@@ -0,0 +1,41 @@
+"""
+delta-engine: declarative schema management for Delta Lake tables.
+
+This is the curated runtime entry point. The preferred user imports are::
+
+    from delta_engine.schema import DeltaTable, Column, Integer
+    from delta_engine.databricks import build_engine
+    from delta_engine import Engine
+
+Only backend-neutral runtime types live here. Schema declarations belong in
+``delta_engine.schema`` and Databricks helpers belong in ``delta_engine.databricks``.
+"""
+
+from importlib.metadata import PackageNotFoundError, version as _version
+
+from delta_engine.application import (
+    Engine,
+    Failure,
+    SyncFailedError,
+    SyncReport,
+    TableRunStatus,
+    render_diff,
+    render_report,
+)
+
+try:
+    __version__ = _version("delta-engine")
+except PackageNotFoundError:  # running from a source tree that is not installed
+    __version__ = "0.0.0"
+
+# `__version__` is package metadata, not part of the runtime API surface, so it
+# is intentionally not advertised in `__all__`; access it as `delta_engine.__version__`.
+__all__ = [
+    "Engine",
+    "Failure",
+    "SyncFailedError",
+    "SyncReport",
+    "TableRunStatus",
+    "render_diff",
+    "render_report",
+]

delta_engine/adapters/databricks/__init__.py

@@ -0,0 +1,8 @@
+"""
+Databricks/Spark adapter package.
+
+Import concrete modules directly (``factory``, ``log_config``, ``reader``,
+``executor``); the public user-facing entry points live in
+:mod:`delta_engine.databricks`. This ``__init__`` stays empty so importing
+the package never pulls PySpark.
+"""

delta_engine/adapters/databricks/errors.py

@@ -0,0 +1,53 @@
+"""
+Translate backend exceptions into neutral summaries for typed failures.
+
+Spark raises a heterogeneous set of failures (``Py4JJavaError``,
+``AnalysisException``, plain Python errors) that varies across runtime
+environments. Both adapter boundaries (reader and executor) reduce any of
+them to the same two facts a failure report needs: the most informative type
+name and a bounded message.
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class ExceptionSummary:
+    """The two facts a typed failure records about a backend exception."""
+
+    type_name: str
+    message: str
+
+
+def summarize_exception(exception: Exception) -> ExceptionSummary:
+    """Summarize an exception as its most informative type name and message head."""
+    return ExceptionSummary(_exception_type_name(exception), _message_preview(exception))
+
+
+def _message_preview(exception: Exception) -> str:
+    """Return the first lines of an exception message, bounded for reports."""
+    message_head = str(exception)
+    return "\n".join(message_head.splitlines()[:5])
+
+
+def _exception_type_name(exception: Exception) -> str:
+    """
+    Return the most informative exception class name available.
+
+    For Py4JJavaError (the primary failure shape on Databricks, where JVM
+    exceptions surface through py4j), the underlying Java class is preferred
+    over the py4j wrapper — e.g. 'org.apache.spark.sql.AnalysisException'
+    rather than 'Py4JJavaError'. Falls back to the Python class name for all
+    other exceptions.
+    """
+    try:
+        from py4j.protocol import Py4JJavaError  # type: ignore[import]
+
+        if isinstance(exception, Py4JJavaError):
+            try:
+                return exception.java_exception.getClass().getName()
+            except (AttributeError, TypeError):
+                return "Py4JJavaError"
+    except ImportError:
+        pass
+    return type(exception).__name__

delta_engine/adapters/databricks/executor.py

@@ -0,0 +1,120 @@
+"""
+Execute compiled plans on Databricks/Spark and capture results.
+
+Compiles an `ActionPlan` to SQL, runs each statement via a `SparkSession`, and
+returns `ExecutionResult` entries including SQL previews and failure details.
+"""
+
+from collections.abc import Iterable
+import logging
+
+from pyspark.sql import SparkSession
+
+from delta_engine.adapters.databricks.errors import summarize_exception
+from delta_engine.adapters.databricks.sql import (
+    CompiledAction,
+    compile_plan,
+)
+from delta_engine.application.failures import ExecutionFailure
+from delta_engine.application.ports import (
+    ExecutionFailed,
+    ExecutionResult,
+    ExecutionSucceeded,
+    ExecutionSummary,
+)
+from delta_engine.domain.model import QualifiedName
+from delta_engine.domain.plan import Action, ActionPlan
+
+logger = logging.getLogger(__name__)
+
+
+class DatabricksExecutor:
+    """Plan executor that runs compiled statements via a Spark session."""
+
+    def __init__(self, spark: SparkSession) -> None:
+        self.spark = spark
+
+    def execute(self, qualified_name: QualifiedName, plan: ActionPlan) -> ExecutionSummary:
+        """
+        Execute the plan's actions against ``qualified_name`` and summarize the outcome.
+
+        Execution stops at the first failure: the actions form a dependency
+        chain, and the engine is not transactional, so continuing past a failure
+        risks compounding a half-migrated table. The summary covers the actions
+        attempted, ending at the one that failed; actions after it are left
+        unattempted rather than run against an inconsistent table.
+        """
+        return _execute_compiled(self.spark, compile_plan(qualified_name, plan))
+
+
+def _execute_compiled(spark: SparkSession, compiled: Iterable[CompiledAction]) -> ExecutionSummary:
+    """
+    Run each compiled action in plan order, stopping at the first failure.
+
+    Holds the stop-on-first-failure loop as a free function so it is testable
+    without a Spark session: a unit test passes a fake ``spark`` and pre-built
+    ``CompiledAction`` pairs, with no need to inject a compiler.
+    """
+    results: list[ExecutionResult] = []
+    for action_index, compiled_action in enumerate(compiled):
+        result = _run_statement(
+            spark, compiled_action.action, action_index, compiled_action.statement
+        )
+        results.append(result)
+        if isinstance(result, ExecutionFailed):
+            break
+    return ExecutionSummary(tuple(results))
+
+
+def _run_statement(
+    spark: SparkSession, action: Action, action_index: int, statement: str
+) -> ExecutionResult:
+    """
+    Run a single compiled statement and map its outcome to an `ExecutionResult`.
+
+    The broad ``except`` is intentional and mirrors the reader's ``fetch_state``:
+    Spark raises a heterogeneous set of failures (``Py4JJavaError``,
+    ``AnalysisException``, and plain Python errors) that varies across runtime
+    environments. The executor's contract is to wrap any failure in an
+    ``ExecutionFailed`` so the run can record it and stop cleanly, never to let
+    a backend-specific exception escape. Narrowing the catch would reintroduce
+    silent propagation of whichever type was missed.
+    """
+    action_name = type(action).__name__
+    preview = _sql_preview(statement)
+    try:
+        spark.sql(statement)
+    except Exception as exception:
+        summary = summarize_exception(exception)
+        logger.warning("%s failed: %s\nSQL: %s", action_name, summary.message, preview)
+        return ExecutionFailed(
+            action=action_name,
+            failure=ExecutionFailure(
+                action_index=action_index,
+                exception_type=summary.type_name,
+                message=summary.message,
+                statement_preview=preview,
+            ),
+        )
+
+    logger.info("Executed: %s", action_name)
+    return ExecutionSucceeded(
+        action=action_name,
+        action_index=action_index,
+        statement_preview=preview,
+    )
+
+
+def _sql_preview(sql: str, *, max_chars: int = 240) -> str:
+    """
+    Return a compact, bounded preview of a SQL statement for logs/results.
+
+    - Normalizes all runs of whitespace to single spaces on one line.
+    - Truncates with an ellipsis when longer than max_chars.
+
+    The bound and formatting are this executor's reporting policy — the preview
+    lands in ``statement_preview`` on execution results and in log lines, never
+    back in SQL sent to Spark.
+    """
+    normalized = " ".join(sql.split())
+    return normalized if len(normalized) <= max_chars else (normalized[:max_chars] + "…")

delta_engine/adapters/databricks/factory.py

@@ -0,0 +1,21 @@
+"""Factory for constructing an `Engine` wired to Databricks/Spark adapters."""
+
+from pyspark.sql import SparkSession
+
+from delta_engine.adapters.databricks.executor import DatabricksExecutor
+from delta_engine.adapters.databricks.reader import DatabricksReader
+from delta_engine.application.engine import Engine
+
+
+def build_engine(spark: SparkSession) -> Engine:
+    """
+    Create an engine configured for Databricks.
+
+    This is the public entry point for end users. It has no logging side effect:
+    for the common script/notebook case, call :func:`configure_logging` once at
+    startup to install the package's coloured handler; embedders who own their
+    own logging simply leave it alone.
+    """
+    reader = DatabricksReader(spark)
+    executor = DatabricksExecutor(spark)
+    return Engine(reader=reader, executor=executor)

delta_engine/adapters/databricks/log_config.py

@@ -0,0 +1,87 @@
+"""
+Logging helpers with ANSI-colored level output.
+
+Provides a `LevelColorFormatter` and `configure_logging` to install a colored
+stderr handler on the root logger for consistent, readable logs across the
+package.
+
+"""
+
+import logging
+import sys
+from typing import TextIO
+
+RESET = "\033[0m"
+YELLOW = "\033[33m"
+RED = "\033[31m"
+BRIGHT_RED = "\033[91m"
+
+
+class LevelColorFormatter(logging.Formatter):
+    """Formatter that colorizes log records based on the level."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Apply ANSI color codes to a formatted log message."""
+        text = super().format(record)
+        if record.levelno == logging.WARNING:
+            return f"{YELLOW}{text}{RESET}"
+        if record.levelno == logging.ERROR:
+            return f"{RED}{text}{RESET}"
+        if record.levelno == logging.CRITICAL:
+            return f"{BRIGHT_RED}{text}{RESET}"
+        return text
+
+
+class SafeStreamHandler(logging.StreamHandler):
+    """
+    StreamHandler that tolerates interpreter/pytest teardown.
+
+    Swallows ValueError raised when the stream is already closed.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Emit a log record, swallowing ValueError if the stream is closed."""
+        try:
+            super().emit(record)
+        except ValueError:
+            pass
+
+
+def configure_logging(level: int = logging.INFO, stream: TextIO | None = None) -> None:
+    """
+    Configure root logging with a colored formatter.
+
+    Takes over the root logger: any handlers already installed on it are
+    removed and replaced with this package's single shutdown-safe coloured
+    handler, and noisy third-party loggers (py4j) are quieted. Intended for
+    scripts and notebooks that want ready-made output; embedders who own
+    their application's logging should configure it themselves and not call
+    this function.
+
+    Args:
+        level: Root log level to set (default ``logging.INFO``).
+        stream: Destination for log records. Defaults to ``sys.__stderr__``,
+            which avoids pytest's captured stream. Notebooks should pass
+            ``sys.stdout`` so logs and ``print`` output share one ordered
+            stream — otherwise stdout and stderr flush independently and log
+            lines can surface after later prints.
+
+    """
+    root = logging.getLogger()
+
+    root.handlers.clear()
+    root.setLevel(level)
+
+    handler = SafeStreamHandler(stream=stream if stream is not None else sys.__stderr__)
+    handler.setFormatter(
+        LevelColorFormatter(
+            "%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+    )
+    root.addHandler(handler)
+
+    for name in ("py4j", "py4j.clientserver"):
+        py4j_logger = logging.getLogger(name)
+        py4j_logger.setLevel(logging.WARNING)
+        py4j_logger.propagate = False