pipeloom-databricks 0.5.0__py3-none-any.whl

pipeloom_databricks/__init__.py

@@ -0,0 +1,18 @@
+"""Databricks renderer for pipeloom: a compiled plan as Jobs settings JSON and bundle YAML.
+
+Rendering is pure. Nothing here imports `databricks.sdk`; only the modules that talk to a
+workspace will, and they load it lazily.
+"""
+
+from __future__ import annotations
+
+
+def pipeloom_databricks_task() -> None:
+    """The entry point of a rendered task, found by `package_name` and `entry_point`.
+
+    A function of the package covers a Databricks lookup that does not read entry points. The
+    task module loads only when a task runs.
+    """
+    from pipeloom_databricks.task import main
+
+    main()

pipeloom_databricks/api.py

@@ -0,0 +1,45 @@
+"""The stable surface other pipeloom distributions (such as an Airflow renderer) may call.
+
+Importing it loads no SDK: `connect` imports `databricks.sdk` only when it runs.
+"""
+
+from __future__ import annotations
+
+from pipeloom_databricks.client import (
+    ApiError,
+    ConnectionNotConfigured,
+    DatabricksClient,
+    JobSummary,
+    RunInfo,
+    TaskRunInfo,
+    connect,
+)
+from pipeloom_databricks.deploy import DeployResult, DuplicateJobs, deploy, find_job
+from pipeloom_databricks.finalize_states import task_states
+from pipeloom_databricks.grouping import task_key_map
+from pipeloom_databricks.render import deploy_options, render_job
+from pipeloom_databricks.repair import repair
+from pipeloom_databricks.run import RunStarted, SelectionError, find_run, run_job
+
+__all__ = [
+    "ApiError",
+    "ConnectionNotConfigured",
+    "DatabricksClient",
+    "DeployResult",
+    "DuplicateJobs",
+    "JobSummary",
+    "RunInfo",
+    "RunStarted",
+    "SelectionError",
+    "TaskRunInfo",
+    "connect",
+    "deploy",
+    "deploy_options",
+    "find_job",
+    "find_run",
+    "render_job",
+    "repair",
+    "run_job",
+    "task_key_map",
+    "task_states",
+]

pipeloom_databricks/bundle.py

@@ -0,0 +1,113 @@
+"""The files of a Declarative Automation Bundle: one job file and one stable manifest per pipeline.
+
+Pure: it returns bytes and never writes. `${workspace.file_path}` is the bundle variable that
+resolves to the synced files at deploy time, so the job finds the project and the manifest
+wherever the bundle lands.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+from pipeloom.diagnostics import Report
+from pipeloom.plan.manifest import manifest_bytes
+from pipeloom.plan.model import Plan
+from pipeloom_databricks.jobs_model import to_json
+from pipeloom_databricks.render import (
+    Locations,
+    _own_version,
+    deploy_options,
+    job_name,
+    render_job,
+)
+from pipeloom_databricks.serialize import to_bundle_yaml
+
+BUNDLE_FILE: Final = "databricks.yml"
+WORKSPACE_FILES: Final = "${workspace.file_path}"
+
+
+@dataclass(frozen=True)
+class BundleFile:
+    path: Path
+    """Where the file goes: `<out>/<name>`."""
+    content: bytes
+
+
+@dataclass(frozen=True)
+class Runtime:
+    """What the target says that the job needs to know."""
+
+    artifacts_root: str | None
+    state_shared: bool
+    cancel_grace: int = 30
+
+
+def find_bundle_root(out: Path, explicit: Path | None) -> Path | None:
+    """`explicit`, else the nearest ancestor of `out` (itself included) that holds databricks.yml."""
+    if explicit is not None:
+        return explicit.resolve()
+    here = out.resolve()
+    for folder in (here, *here.parents):
+        if (folder / BUNDLE_FILE).is_file():
+            return folder
+    return None
+
+
+def relative_to_root(path: Path, root: Path) -> str | None:
+    """`path` below `root` as POSIX text (empty for the root itself), or None when outside."""
+    try:
+        rel = path.resolve().relative_to(root).as_posix()
+    except ValueError:
+        return None
+    return "" if rel == "." else rel
+
+
+def _workspace(rel: str) -> str:
+    return f"{WORKSPACE_FILES}/{rel}" if rel else WORKSPACE_FILES
+
+
+def render_bundle(
+    plan: Plan,
+    runtime: Runtime,
+    pipelines: Sequence[str],
+    *,
+    out: Path,
+    project_rel: str,
+    out_rel: str,
+    report: Report,
+) -> list[BundleFile] | None:
+    """The files of the given pipelines (all of them when empty), or None with the problems in
+    `report`. `project_rel` and `out_rel` are the project folder and the output folder relative
+    to the bundle root."""
+    opts = deploy_options(plan, report)
+    if opts is None:
+        return None
+    manifest = manifest_bytes(plan, stable=True)
+    files: list[BundleFile] = []
+    before = report.error_count
+    for pipeline in pipelines or sorted(opts.pipelines):
+        name = job_name(plan, pipeline)
+        locs = Locations(
+            project_root=_workspace(project_rel),
+            manifest_path=_workspace(f"{out_rel}/{name}.manifest.json".lstrip("/")),
+            artifacts_root=runtime.artifacts_root,
+            state_shared=runtime.state_shared,
+            cancel_grace=runtime.cancel_grace,
+        )
+        job = render_job(plan, pipeline, locs, report)
+        if job is None:
+            continue
+        header = (
+            f"Generated by pipeloom-databricks {_own_version(plan)}.\n"
+            f"Project: {plan.metadata.project}. Pipeline: {pipeline}. "
+            f"Target: {plan.metadata.target}.\n"
+            "Do not edit: run `pipeloom databricks bundle` again."
+        )
+        files.append(
+            BundleFile(out / f"{name}.job.yml", to_bundle_yaml(name, to_json(job.settings), header))
+        )
+        files.append(BundleFile(out / f"{name}.manifest.json", manifest))
+    return None if report.error_count > before else files