vibedata-dlt-databricks-notebook 0.1.0__tar.gz

vibedata_dlt_databricks_notebook-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+.pytest_cache/
+uv.lock

vibedata_dlt_databricks_notebook-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.5
+Name: vibedata-dlt-databricks-notebook
+Version: 0.1.0
+Summary: VibeData dlt runtime — Databricks notebook execution unit (runs inside a Databricks notebook): a dlt provider for the Domain's Databricks secret scope, and a databricks destination that stages load files through the Files API. dbutils is provided by the Databricks runtime.
+Author-email: Vibedata <eng@acceleratedata.ai>
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: dlt[databricks]<1.31,>=1.30.0
+Requires-Dist: vibedata-dlt-core>=0.11.0
+Provides-Extra: dev
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# vibedata-dlt-databricks-notebook
+
+Databricks notebook execution unit for the VibeData dlt runtime — the installed distribution for Databricks domains running a dlt pipeline inside a Databricks notebook. `dbutils` is provided by the Databricks runtime, not a pip dependency.
+
+```python
+from vibedata.dlt.databricks import DatabricksSecretScopeProvider, databricks_files_api, setup_environment
+import dlt
+
+setup_environment()
+dlt.config.register_provider(DatabricksSecretScopeProvider())
+pipeline = dlt.pipeline(pipeline_name="<name>", destination=databricks_files_api(credentials={...}), dataset_name="<schema>")
+```
+
+- **Import order.** Import this package before anything imports `dlt`: on Databricks compute `import dlt` otherwise binds Databricks' own Lakeflow Declarative Pipelines module.
+- **`setup_environment()`** makes the Domain's `ingestion/` (the notebook's `ingestion_dir` widget) the working directory and an import root, names dlt's scratch schema `<dataset>_dlt_staging` so it cannot collide with a dbt-modelled `staging` layer, and caps load files below the SDK's single-upload limit.
+- **`DatabricksSecretScopeProvider`** resolves dlt secrets from the Databricks secret scope that `[providers.databricks_secret_scope] scope` names in the committed `.dlt/config.toml`. Each key is the dotted dlt key, lowercased. It resolves nothing when no scope is configured.
+- **`databricks_files_api`** is dlt's `databricks` destination with one change: it stages load files on the volume through the Files API instead of the SQL connector's `PUT`. For a catalog on Databricks-managed storage, `PUT` uploads to the region's storage gateway, whose name resolves to a refusing private address on serverless compute. dlt has no hook for the upload, so the class overrides dlt's private staging methods; raise the package's dlt cap only after re-checking them against the new release.
+- **No `finalize()`.** The unit writes no audit tables yet, so a Databricks pipeline makes the one `setup_environment()` call; the other units' audit pair arrives with Databricks run history.
+
+Install this package; it pulls in `vibedata-dlt-core` and `dlt[databricks]`.

vibedata_dlt_databricks_notebook-0.1.0/README.md

@@ -0,0 +1,20 @@
+# vibedata-dlt-databricks-notebook
+
+Databricks notebook execution unit for the VibeData dlt runtime — the installed distribution for Databricks domains running a dlt pipeline inside a Databricks notebook. `dbutils` is provided by the Databricks runtime, not a pip dependency.
+
+```python
+from vibedata.dlt.databricks import DatabricksSecretScopeProvider, databricks_files_api, setup_environment
+import dlt
+
+setup_environment()
+dlt.config.register_provider(DatabricksSecretScopeProvider())
+pipeline = dlt.pipeline(pipeline_name="<name>", destination=databricks_files_api(credentials={...}), dataset_name="<schema>")
+```
+
+- **Import order.** Import this package before anything imports `dlt`: on Databricks compute `import dlt` otherwise binds Databricks' own Lakeflow Declarative Pipelines module.
+- **`setup_environment()`** makes the Domain's `ingestion/` (the notebook's `ingestion_dir` widget) the working directory and an import root, names dlt's scratch schema `<dataset>_dlt_staging` so it cannot collide with a dbt-modelled `staging` layer, and caps load files below the SDK's single-upload limit.
+- **`DatabricksSecretScopeProvider`** resolves dlt secrets from the Databricks secret scope that `[providers.databricks_secret_scope] scope` names in the committed `.dlt/config.toml`. Each key is the dotted dlt key, lowercased. It resolves nothing when no scope is configured.
+- **`databricks_files_api`** is dlt's `databricks` destination with one change: it stages load files on the volume through the Files API instead of the SQL connector's `PUT`. For a catalog on Databricks-managed storage, `PUT` uploads to the region's storage gateway, whose name resolves to a refusing private address on serverless compute. dlt has no hook for the upload, so the class overrides dlt's private staging methods; raise the package's dlt cap only after re-checking them against the new release.
+- **No `finalize()`.** The unit writes no audit tables yet, so a Databricks pipeline makes the one `setup_environment()` call; the other units' audit pair arrives with Databricks run history.
+
+Install this package; it pulls in `vibedata-dlt-core` and `dlt[databricks]`.

vibedata_dlt_databricks_notebook-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vibedata-dlt-databricks-notebook"
+version = "0.1.0"
+description = "VibeData dlt runtime — Databricks notebook execution unit (runs inside a Databricks notebook): a dlt provider for the Domain's Databricks secret scope, and a databricks destination that stages load files through the Files API. dbutils is provided by the Databricks runtime."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Vibedata", email = "eng@acceleratedata.ai" }]
+
+dependencies = [
+    "vibedata-dlt-core>=0.11.0",
+    "dlt[databricks]>=1.30.0,<1.31",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "pytest-mock>=3.12"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vibedata"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

vibedata_dlt_databricks_notebook-0.1.0/src/vibedata/dlt/databricks/__init__.py

@@ -0,0 +1,25 @@
+"""VibeData dlt runtime — Databricks notebook execution unit.
+
+The installed unit for Databricks domains running inside a Databricks notebook:
+
+    from vibedata.dlt.databricks import DatabricksSecretScopeProvider, databricks_files_api, setup_environment
+    import dlt
+
+    setup_environment()
+    dlt.config.register_provider(DatabricksSecretScopeProvider())
+    pipeline = dlt.pipeline(pipeline_name="<name>", destination=databricks_files_api(...), dataset_name="<schema>")
+
+Import this package before anything imports `dlt`: on Databricks compute, `import dlt`
+otherwise binds Databricks' own Lakeflow Declarative Pipelines module, and this package
+imports dlthub's in its place. `dbutils` is provided by the Databricks runtime.
+"""
+
+from ._dlt_import import import_dlthub_dlt
+
+import_dlthub_dlt()
+
+from .bootstrap import setup_environment  # noqa: E402
+from .destination import databricks_files_api  # noqa: E402
+from .providers.secret_scope import DatabricksSecretScopeProvider  # noqa: E402
+
+__all__ = ["DatabricksSecretScopeProvider", "databricks_files_api", "setup_environment"]

vibedata_dlt_databricks_notebook-0.1.0/src/vibedata/dlt/databricks/_dlt_import.py

@@ -0,0 +1,36 @@
+"""Import dlthub's `dlt` on Databricks compute.
+
+Databricks registers a post-import hook that answers `import dlt` with its own Lakeflow
+Declarative Pipelines module. Importing with that hook set aside binds dlthub's package,
+the approach dlt's own Databricks destination docs give for serverless compute.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from types import ModuleType
+
+POST_IMPORT_HOOK = "PostImportHook"
+
+
+def _is_dlthub(module: ModuleType) -> bool:
+    return hasattr(module, "pipeline")
+
+
+def import_dlthub_dlt() -> ModuleType:
+    """Return dlthub's `dlt`, importing it past Databricks' hook when nothing has yet."""
+    loaded = sys.modules.get("dlt")
+    if loaded is None:
+        hooks = list(sys.meta_path)
+        sys.meta_path = [hook for hook in hooks if POST_IMPORT_HOOK not in repr(hook)]
+        try:
+            loaded = importlib.import_module("dlt")
+        finally:
+            sys.meta_path = hooks
+    if not _is_dlthub(loaded):
+        raise ImportError(
+            "`dlt` resolved to Databricks' Lakeflow Declarative Pipelines module, not dlthub's. "
+            "Import vibedata.dlt.databricks before anything imports dlt."
+        )
+    return loaded

vibedata_dlt_databricks_notebook-0.1.0/src/vibedata/dlt/databricks/bootstrap.py

@@ -0,0 +1,53 @@
+"""Prepare a Databricks notebook for a dlt run of the Domain's committed pipeline."""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import Optional
+
+INGESTION_DIR_WIDGET = "ingestion_dir"
+
+# dlt's default staging layout, `%s_staging`, is the schema dbt derives for a `staging`
+# custom schema, so the two would share one schema.
+STAGING_LAYOUT_KEY = "DESTINATION__STAGING_DATASET_NAME_LAYOUT"
+STAGING_LAYOUT = "%s_dlt_staging"
+
+# databricks_files_api uploads a load file in one Files API call only below the SDK's
+# multipart threshold (50 MiB); a larger file goes through the unreachable storage gateway.
+FILE_MAX_BYTES_KEY = "DATA_WRITER__FILE_MAX_BYTES"
+FILE_MAX_BYTES = 40 * 1024 * 1024
+
+
+def setup_environment(ingestion_dir: Optional[str] = None) -> None:
+    """Run from the Domain's `ingestion/`, so `sources.*` imports and `.dlt/` resolve.
+
+    `ingestion_dir` defaults to the notebook's `ingestion_dir` widget, which the job sets.
+    """
+    if ingestion_dir is None:
+        from databricks.sdk.runtime import dbutils
+
+        ingestion_dir = dbutils.widgets.get(INGESTION_DIR_WIDGET)
+    file_max_bytes = _file_max_bytes(os.environ.get(FILE_MAX_BYTES_KEY))
+    ingestion_dir = os.path.abspath(ingestion_dir)
+    os.chdir(ingestion_dir)
+    if ingestion_dir not in sys.path:
+        sys.path.insert(0, ingestion_dir)
+    os.environ[STAGING_LAYOUT_KEY] = STAGING_LAYOUT
+    os.environ[FILE_MAX_BYTES_KEY] = str(file_max_bytes)
+
+
+def _file_max_bytes(configured: Optional[str]) -> int:
+    """The cap for `DATA_WRITER__FILE_MAX_BYTES`, the one setting this guards.
+
+    A lower value already in that variable is kept; unset, empty, 0 (dlt's "unlimited"), negative
+    or higher gets FILE_MAX_BYTES. dlt rotates files at buffer flushes, so the cap bounds a file's
+    size closely, not exactly; FILE_MAX_BYTES leaves the margin below the SDK's threshold.
+    """
+    if not configured:
+        return FILE_MAX_BYTES
+    try:
+        value = int(configured)
+    except ValueError:
+        raise ValueError(f"{FILE_MAX_BYTES_KEY} must be a whole number of bytes; got {configured!r}") from None
+    return FILE_MAX_BYTES if value <= 0 else min(value, FILE_MAX_BYTES)

vibedata_dlt_databricks_notebook-0.1.0/src/vibedata/dlt/databricks/destination.py

@@ -0,0 +1,76 @@
+"""A `databricks` destination that stages load files through the Files API.
+
+dlt stages each local load file on a Unity Catalog volume with the SQL connector's `PUT`,
+which uploads to a signed URL. For a catalog on Databricks-managed storage that URL is on
+the region's storage gateway, and on serverless compute the gateway's name resolves to a
+private address that refuses the connection. The Files API reaches the same volume
+through the workspace host. Everything else is dlt's own: the volume, the file layout and
+the `COPY INTO` that loads it.
+
+dlt has no hook for the upload, so this overrides the two private methods that stage and
+remove the file. The package caps dlt to the minor release they were checked against.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from databricks.sdk import WorkspaceClient
+from dlt.common.storages import FileStorage
+from dlt.common.utils import uniq_id
+from dlt.destinations import databricks
+from dlt.destinations.impl.databricks.databricks import DatabricksClient, DatabricksLoadJob
+
+DEFAULT_STAGING_VOLUME = "_dlt_staging_load_volume"
+
+
+def _files() -> Any:
+    return WorkspaceClient().files
+
+
+class FilesApiLoadJob(DatabricksLoadJob):
+    def _handle_local_file_upload(self, local_file_path: str) -> tuple[str, str, str, str]:
+        file_name = FileStorage.get_file_name_from_file_path(local_file_path)
+        # Databricks refuses to load a file whose name starts with `_` or `.`.
+        volume_file_name = "valid" + file_name if file_name.startswith(("_", ".")) else file_name
+        config = self._job_client.config
+        if config.staging_volume_name:
+            catalog, schema, volume = config.staging_volume_name.split(".")
+        else:
+            catalog, schema, volume = self._sql_client.database_name, self._sql_client.dataset_name, DEFAULT_STAGING_VOLUME
+            self._sql_client.execute_sql(f"CREATE VOLUME IF NOT EXISTS {catalog}.{schema}.{volume}")
+        volume_path = f"/Volumes/{catalog}/{schema}/{volume}/{uniq_id()}"
+        volume_file_path = f"{volume_path}/{volume_file_name}"
+        with open(local_file_path, "rb") as source:
+            _files().upload(volume_file_path, source, overwrite=True)
+        return f"FROM '{volume_path}'", file_name, volume_path, volume_file_path
+
+    def _handle_staged_file_remove(self, volume_path: str, volume_file_path: str) -> None:
+        files = _files()
+        files.delete(volume_file_path)
+        files.delete_directory(volume_path)
+
+
+class FilesApiDatabricksClient(DatabricksClient):
+    def create_load_job(self, table: Any, file_path: str, load_id: str, restore: bool = False) -> Any:
+        job = super().create_load_job(table, file_path, load_id, restore)
+        # dlt builds this job only for the exact class, so it is swapped after the fact;
+        # reference, merge and Zerobus jobs stay as dlt built them.
+        if type(job) is DatabricksLoadJob:
+            job = FilesApiLoadJob(file_path, staging_config=job._staging_config)
+        return job
+
+
+class databricks_files_api(databricks):  # noqa: N801 - dlt names destination factories in lowercase
+    """dlt's `databricks` destination, staging through the Files API.
+
+    Named `databricks` by default, so `[destination.databricks]` config still applies.
+    """
+
+    def __init__(self, *args: Any, destination_name: Optional[str] = None, **kwargs: Any) -> None:
+        # The default stays None: dlt drops an argument equal to its signature default as unset.
+        super().__init__(*args, destination_name=destination_name or "databricks", **kwargs)
+
+    @property
+    def client_class(self) -> type[DatabricksClient]:
+        return FilesApiDatabricksClient

vibedata_dlt_databricks_notebook-0.1.0/src/vibedata/dlt/databricks/providers/secret_scope.py

@@ -0,0 +1,105 @@
+"""The Domain's Databricks secret scope as a dlt secrets provider.
+
+A `databricks_secret_scope` Secret Store keeps each credential as a key in one
+Databricks-backed secret scope, named by its dotted dlt key, lowercased
+(`sources.sf_prod.credentials.client_id`). The committed `.dlt/config.toml` names the scope:
+
+    [providers.databricks_secret_scope]
+    scope = "<scope>"
+"""
+
+from __future__ import annotations
+
+import logging
+import tomllib
+from pathlib import Path
+from typing import Any, Optional, Sequence, Tuple, Type
+
+from dlt.common.configuration.providers import ConfigProvider
+
+logger = logging.getLogger(__name__)
+
+PROVIDER_NAME = "databricks_secret_scope"
+CONFIG_FILE = Path(".dlt") / "config.toml"
+
+# dlt retries a key with its sections popped from the right, down to none. Below two
+# (`sources` and the connection) a key names nothing connection-specific, and answering
+# it would serve one source another's credential.
+MINIMUM_SECTIONS = 2
+MAX_KEY_LENGTH = 128
+
+
+def scope_key(key: str, *sections: str) -> str:
+    """The secret scope key for a dlt key. Raises for a path that cannot be one."""
+    if len(sections) < MINIMUM_SECTIONS or not all(sections):
+        raise ValueError(f"a scope key needs at least {MINIMUM_SECTIONS} sections, none empty; got {sections!r}")
+    name = ".".join((*sections, key)).lower()
+    if len(name) > MAX_KEY_LENGTH:
+        raise ValueError(f"{name!r} is longer than a secret scope key's {MAX_KEY_LENGTH} characters")
+    return name
+
+
+def configured_scope(config_file: Path = CONFIG_FILE) -> Optional[str]:
+    """The scope the committed config names, or None when it names none."""
+    if not config_file.is_file():
+        return None
+    config = tomllib.loads(config_file.read_text(encoding="utf-8"))
+    return config.get("providers", {}).get(PROVIDER_NAME, {}).get("scope") or None
+
+
+class DatabricksSecretScopeProvider(ConfigProvider):
+    """Resolve dlt secrets from the Domain's Databricks secret scope.
+
+    Construct it after `setup_environment()`, which makes `ingestion/` the working directory
+    its config is read from. With no scope configured it resolves nothing.
+    """
+
+    def __init__(self, scope: Optional[str] = None, secrets: Any = None) -> None:
+        self._scope = scope if scope is not None else configured_scope()
+        self._secrets = secrets
+        self._keys: Optional[set[str]] = None
+        if not self._scope:
+            logger.info("No [providers.%s] scope configured; resolving no secrets from it", PROVIDER_NAME)
+
+    @property
+    def name(self) -> str:
+        return PROVIDER_NAME
+
+    @property
+    def supports_secrets(self) -> bool:
+        return True
+
+    @property
+    def supports_sections(self) -> bool:
+        return True
+
+    @property
+    def locations(self) -> Sequence[str]:
+        # dlt renders locations through `os.path.abspath` in a resolution diagnostic.
+        return []
+
+    def get_value(
+        self, key: str, hint: Type[Any], pipeline_name: Optional[str], *sections: str
+    ) -> Tuple[Optional[Any], str]:
+        del hint, pipeline_name
+        try:
+            name = scope_key(key, *sections)
+        except ValueError:
+            return None, f"<no secret scope key for sections {sections!r}>"
+        if not self._scope or name not in self._available_keys():
+            return None, name
+        return self._client().get(self._scope, name), name
+
+    def _client(self) -> Any:
+        if self._secrets is None:
+            from databricks.sdk.runtime import dbutils
+
+            self._secrets = dbutils.secrets
+        return self._secrets
+
+    def _available_keys(self) -> set[str]:
+        # Listed once: `get` on an absent key raises a generic runtime error, which would
+        # leave a missing secret indistinguishable from a failing scope.
+        if self._keys is None:
+            self._keys = {secret.key for secret in self._client().list(self._scope)}
+        return self._keys

vibedata_dlt_databricks_notebook-0.1.0/tests/test_bootstrap.py

@@ -0,0 +1,92 @@
+import os
+import sys
+import types
+
+import pytest
+
+from vibedata.dlt.databricks import _dlt_import, bootstrap
+
+
+@pytest.fixture(autouse=True)
+def restore_environ(monkeypatch):
+    # setup_environment writes os.environ; snapshot it so no test leaks into the next.
+    monkeypatch.setattr(os, "environ", os.environ.copy())
+
+
+@pytest.mark.parametrize("configured, expected", [("0", bootstrap.FILE_MAX_BYTES), ("-5", bootstrap.FILE_MAX_BYTES), (str(10**12), bootstrap.FILE_MAX_BYTES)])
+def test_an_unlimited_or_higher_file_cap_gets_the_upload_limit(configured, expected):
+    assert bootstrap._file_max_bytes(configured) == expected
+
+
+def test_a_file_cap_that_is_not_a_number_names_the_variable():
+    with pytest.raises(ValueError, match=bootstrap.FILE_MAX_BYTES_KEY):
+        bootstrap._file_max_bytes("40MB")
+
+
+def test_setup_environment_runs_from_the_ingestion_dir(tmp_path, monkeypatch):
+    monkeypatch.chdir(os.getcwd())
+    monkeypatch.setattr(sys, "path", list(sys.path))
+    monkeypatch.delenv(bootstrap.STAGING_LAYOUT_KEY, raising=False)
+    monkeypatch.delenv(bootstrap.FILE_MAX_BYTES_KEY, raising=False)
+
+    bootstrap.setup_environment(str(tmp_path))
+
+    assert os.getcwd() == str(tmp_path)
+    assert sys.path[0] == str(tmp_path)
+    assert os.environ[bootstrap.STAGING_LAYOUT_KEY] == "%s_dlt_staging"
+    assert int(os.environ[bootstrap.FILE_MAX_BYTES_KEY]) < 50 * 1024 * 1024
+
+
+def test_import_returns_dlthub_dlt_when_already_loaded():
+    import dlt
+
+    assert _dlt_import.import_dlthub_dlt() is dlt
+
+
+def test_import_refuses_databricks_own_dlt(monkeypatch):
+    monkeypatch.setitem(sys.modules, "dlt", types.ModuleType("dlt"))
+
+    with pytest.raises(ImportError, match="Lakeflow"):
+        _dlt_import.import_dlthub_dlt()
+
+
+def test_import_sets_the_post_import_hook_aside_only_for_the_import(monkeypatch):
+    class PostImportHook:
+        def find_spec(self, *args, **kwargs):
+            raise AssertionError("the hook answered the import")
+
+    hook = PostImportHook()
+    seen = []
+    dlt_module = sys.modules["dlt"]
+    monkeypatch.delitem(sys.modules, "dlt")
+    monkeypatch.setattr(sys, "meta_path", [hook, *sys.meta_path])
+
+    def fake_import(name):
+        seen.append(list(sys.meta_path))
+        return dlt_module
+
+    monkeypatch.setattr(_dlt_import.importlib, "import_module", fake_import)
+
+    assert _dlt_import.import_dlthub_dlt() is dlt_module
+    assert hook not in seen[0]
+    assert sys.meta_path[0] is hook
+
+
+def test_a_lower_file_cap_the_domain_set_is_kept(tmp_path, monkeypatch):
+    monkeypatch.chdir(os.getcwd())
+    monkeypatch.setattr(sys, "path", list(sys.path))
+    monkeypatch.setenv(bootstrap.FILE_MAX_BYTES_KEY, "1000")
+
+    bootstrap.setup_environment(str(tmp_path))
+
+    assert os.environ[bootstrap.FILE_MAX_BYTES_KEY] == "1000"
+
+
+def test_a_relative_ingestion_dir_stays_importable(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(sys, "path", list(sys.path))
+    (tmp_path / "ingestion").mkdir()
+
+    bootstrap.setup_environment("ingestion")
+
+    assert sys.path[0] == str(tmp_path / "ingestion")

vibedata_dlt_databricks_notebook-0.1.0/tests/test_destination.py

@@ -0,0 +1,119 @@
+from types import SimpleNamespace
+
+import pytest
+
+from vibedata.dlt.databricks import destination
+from vibedata.dlt.databricks.destination import (
+    DatabricksLoadJob,
+    FilesApiDatabricksClient,
+    FilesApiLoadJob,
+    databricks_files_api,
+)
+
+
+class FakeFiles:
+    def __init__(self):
+        self.calls = []
+
+    def upload(self, path, source, overwrite):
+        self.calls.append(("upload", path, source.read(), overwrite))
+
+    def delete(self, path):
+        self.calls.append(("delete", path))
+
+    def delete_directory(self, path):
+        self.calls.append(("delete_directory", path))
+
+
+class FakeSqlClient:
+    database_name = "cat"
+    dataset_name = "raw"
+
+    def __init__(self):
+        self.statements = []
+
+    def execute_sql(self, sql):
+        self.statements.append(sql)
+
+
+@pytest.fixture
+def files(monkeypatch):
+    fake = FakeFiles()
+    monkeypatch.setattr(destination, "_files", lambda: fake)
+    monkeypatch.setattr(destination, "uniq_id", lambda: "u1")
+    return fake
+
+
+def load_job(staging_volume_name=None):
+    job = object.__new__(FilesApiLoadJob)
+    job._job_client = SimpleNamespace(config=SimpleNamespace(staging_volume_name=staging_volume_name))
+    job._sql_client = FakeSqlClient()
+    return job
+
+
+def test_upload_goes_through_the_files_api_into_the_named_volume(tmp_path, files):
+    local = tmp_path / "pokemon.abc.0.parquet"
+    local.write_bytes(b"rows")
+    job = load_job("cat.raw.vol")
+
+    result = job._handle_local_file_upload(str(local))
+
+    assert files.calls == [("upload", "/Volumes/cat/raw/vol/u1/pokemon.abc.0.parquet", b"rows", True)]
+    assert result == ("FROM '/Volumes/cat/raw/vol/u1'", "pokemon.abc.0.parquet", "/Volumes/cat/raw/vol/u1", "/Volumes/cat/raw/vol/u1/pokemon.abc.0.parquet")
+    assert job._sql_client.statements == []
+
+
+def test_without_a_named_volume_it_creates_dlts_default_one(tmp_path, files):
+    local = tmp_path / "_dlt_pipeline_state.abc.0.parquet"
+    local.write_bytes(b"state")
+    job = load_job()
+
+    _, file_name, _, volume_file_path = job._handle_local_file_upload(str(local))
+
+    assert job._sql_client.statements == ["CREATE VOLUME IF NOT EXISTS cat.raw._dlt_staging_load_volume"]
+    assert file_name == "_dlt_pipeline_state.abc.0.parquet"
+    assert volume_file_path == "/Volumes/cat/raw/_dlt_staging_load_volume/u1/valid_dlt_pipeline_state.abc.0.parquet"
+
+
+def test_remove_deletes_the_file_then_its_folder(files):
+    load_job()._handle_staged_file_remove("/Volumes/c/s/v/u1", "/Volumes/c/s/v/u1/f.parquet")
+
+    assert files.calls == [("delete", "/Volumes/c/s/v/u1/f.parquet"), ("delete_directory", "/Volumes/c/s/v/u1")]
+
+
+def test_the_client_swaps_only_dlts_own_copy_into_job(monkeypatch):
+    staging_config = object()
+    built = object.__new__(DatabricksLoadJob)
+    built._staging_config = staging_config
+    other = object()
+    jobs = iter([built, other])
+    monkeypatch.setattr(destination.DatabricksClient, "create_load_job", lambda *a, **k: next(jobs))
+    monkeypatch.setattr(FilesApiLoadJob, "__init__", lambda self, path, staging_config: setattr(self, "args", (path, staging_config)))
+    client = object.__new__(FilesApiDatabricksClient)
+
+    swapped = client.create_load_job({}, "t.abc.0.parquet", "load")
+
+    assert type(swapped) is FilesApiLoadJob
+    assert swapped.args == ("t.abc.0.parquet", staging_config)
+    assert client.create_load_job({}, "t.abc.0.parquet", "load") is other
+
+
+def test_the_destination_keeps_dlts_databricks_name_and_config_section():
+    factory = databricks_files_api(credentials={"catalog": "cat"})
+
+    assert factory.destination_name == "databricks"
+    assert factory.client_class is FilesApiDatabricksClient
+    assert databricks_files_api(destination_name="other").destination_name == "other"
+
+
+@pytest.mark.parametrize("method", ["_handle_local_file_upload", "_handle_staged_file_remove"])
+def test_the_overridden_methods_still_exist_on_dlts_job(method):
+    # The overrides only take effect while dlt calls these names; a dlt release that renames
+    # one would silently send the upload back through PUT.
+    assert method in vars(DatabricksLoadJob)
+
+
+def test_the_files_api_job_builds_with_dlts_constructor(tmp_path):
+    job = FilesApiLoadJob(str(tmp_path / "pokemon.abc.0.parquet"), staging_config=None)
+
+    assert job._staging_config is None

vibedata_dlt_databricks_notebook-0.1.0/tests/test_secret_scope.py

@@ -0,0 +1,93 @@
+from types import SimpleNamespace
+
+import pytest
+
+from vibedata.dlt.databricks.providers.secret_scope import (
+    DatabricksSecretScopeProvider,
+    configured_scope,
+    scope_key,
+)
+
+
+class FakeSecrets:
+    def __init__(self, values):
+        self.values = values
+        self.gets = []
+
+    def list(self, scope):
+        return [SimpleNamespace(key=key) for (s, key) in self.values if s == scope]
+
+    def get(self, scope, key):
+        self.gets.append(key)
+        return self.values[(scope, key)]
+
+
+def test_scope_key_is_the_dotted_key_lowercased():
+    assert scope_key("Client_ID", "sources", "SF_prod", "credentials") == "sources.sf_prod.credentials.client_id"
+
+
+@pytest.mark.parametrize("sections", [(), ("sources",), ("sources", "")])
+def test_scope_key_refuses_a_path_that_names_no_connection(sections):
+    with pytest.raises(ValueError):
+        scope_key("password", *sections)
+
+
+def test_scope_key_refuses_a_name_longer_than_a_scope_key():
+    with pytest.raises(ValueError):
+        scope_key("k", "sources", "x" * 130)
+
+
+def test_resolves_a_key_present_in_the_scope():
+    secrets = FakeSecrets({("domain", "sources.crm.credentials.password"): "s3cret"})
+    provider = DatabricksSecretScopeProvider(scope="domain", secrets=secrets)
+
+    assert provider.get_value("password", str, None, "sources", "crm", "credentials") == (
+        "s3cret",
+        "sources.crm.credentials.password",
+    )
+
+
+def test_an_absent_key_resolves_to_none_without_a_get():
+    secrets = FakeSecrets({})
+    provider = DatabricksSecretScopeProvider(scope="domain", secrets=secrets)
+
+    assert provider.get_value("password", str, None, "sources", "crm")[0] is None
+    assert secrets.gets == []
+
+
+def test_a_shallow_path_is_never_answered():
+    secrets = FakeSecrets({("domain", "sources.password"): "other"})
+    provider = DatabricksSecretScopeProvider(scope="domain", secrets=secrets)
+
+    assert provider.get_value("password", str, None, "sources")[0] is None
+
+
+def test_no_configured_scope_resolves_nothing(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    provider = DatabricksSecretScopeProvider(secrets=FakeSecrets({}))
+
+    assert provider.get_value("password", str, None, "sources", "crm")[0] is None
+
+
+def test_configured_scope_reads_the_committed_config(tmp_path):
+    config = tmp_path / "config.toml"
+    config.write_text('[providers.databricks_secret_scope]\nscope = "domain"\n')
+
+    assert configured_scope(config) == "domain"
+    assert configured_scope(tmp_path / "absent.toml") is None
+
+
+def test_the_scope_is_listed_once():
+    calls = []
+
+    class CountingSecrets(FakeSecrets):
+        def list(self, scope):
+            calls.append(scope)
+            return super().list(scope)
+
+    secrets = CountingSecrets({("domain", "sources.crm.credentials.password"): "s3cret"})
+    provider = DatabricksSecretScopeProvider(scope="domain", secrets=secrets)
+    provider.get_value("password", str, None, "sources", "crm", "credentials")
+    provider.get_value("user", str, None, "sources", "crm", "credentials")
+
+    assert calls == ["domain"]