dvc-databricks 1.0.3__tar.gz

dvc_databricks-1.0.3/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint-and-build:
+    name: Lint & Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install hatchling
+
+      - name: Check package builds cleanly
+        run: python -m hatchling build
+
+      - name: Verify package installs and registers dbvol://
+        run: |
+          pip install dist/*.whl
+          python -c "
+          from dvc_objects.fs import known_implementations
+          assert 'dbvol' in known_implementations, 'dbvol not registered!'
+          print('dbvol registered successfully:', known_implementations['dbvol'])
+          "

dvc_databricks-1.0.3/.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    name: Semantic Release & PyPI Publish
+    runs-on: ubuntu-latest
+    concurrency: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install tools
+        run: pip install python-semantic-release hatchling twine
+
+      - name: Run semantic release (version bump + GitHub release)
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          semantic-release version
+
+      - name: Build package
+        run: python -m hatchling build
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: twine upload dist/*

dvc_databricks-1.0.3/.gitignore

@@ -0,0 +1,211 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# DVC local cache and temp files (never committed)
+.dvc/cache
+.dvc/tmp

dvc_databricks-1.0.3/CHANGELOG.md

@@ -0,0 +1,31 @@
+# CHANGELOG
+
+<!-- version list -->
+
+## v1.0.3 (2026-04-03)
+
+### Bug Fixes
+
+- Remove broken output command from release workflow
+  ([`d91f6a1`](https://github.com/ogreyesp/dvc-databricks/commit/d91f6a1b07747430091c69689104cc27c9ddea56))
+
+
+## v1.0.2 (2026-04-03)
+
+### Bug Fixes
+
+- Separate CI from release workflow and use twine for PyPI upload
+  ([`3f8783d`](https://github.com/ogreyesp/dvc-databricks/commit/3f8783dc260a6f5d965bdde48d8ede4bd83718d3))
+
+
+## v1.0.1 (2026-04-03)
+
+### Bug Fixes
+
+- Include .pth file in wheel root via force-include
+  ([`43637ea`](https://github.com/ogreyesp/dvc-databricks/commit/43637ea1da0903c797e18a2e7f3ebd18b7fdb66d))
+
+
+## v1.0.0 (2026-04-03)
+
+- Initial Release

dvc_databricks-1.0.3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Oscar Gabriel Reyes Pupo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dvc_databricks-1.0.3/PKG-INFO

@@ -0,0 +1,213 @@
+Metadata-Version: 2.4
+Name: dvc-databricks
+Version: 1.0.3
+Summary: DVC remote plugin for Databricks Unity Catalog Volumes
+Project-URL: Homepage, https://github.com/ogreyesp/dvc-databricks
+Project-URL: Repository, https://github.com/ogreyesp/dvc-databricks
+Project-URL: Issues, https://github.com/ogreyesp/dvc-databricks/issues
+Author: Óscar Reyes
+License: MIT License
+        
+        Copyright (c) 2026 Oscar Gabriel Reyes Pupo
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: data-versioning,databricks,dvc,mlops,unity-catalog
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: databricks-sdk>=0.40.0
+Requires-Dist: dvc-objects>=5.0
+Requires-Dist: dvc>=3.0
+Requires-Dist: fsspec
+Description-Content-Type: text/markdown
+
+# dvc-databricks
+
+A [DVC](https://dvc.org) remote storage plugin that enables data versioning on **Databricks Unity Catalog Volumes**.
+
+Store large data files on Databricks Volumes (backed by S3 or ADLS), keep only lightweight `.dvc` pointer files in your git repository, and use standard DVC commands — no custom code required.
+
+```bash
+dvc push   # uploads data to Databricks Volume via Databricks SDK
+dvc pull   # downloads data from Databricks Volume
+```
+
+---
+
+## Why this plugin?
+
+Databricks Unity Catalog Volumes cannot be accessed like a plain S3 bucket — all I/O should go through the **Databricks Files API**. This plugin bridges DVC and the Databricks SDK so you can version and share datasets stored on Volumes without ever leaving the standard DVC workflow.
+
+---
+
+## Requirements
+
+- Python >= 3.10
+- [DVC](https://dvc.org/doc/install) >= 3.0
+- [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/install.html) configured with a profile in `~/.databrickscfg`
+- Access to a Databricks Unity Catalog Volume
+
+---
+
+## Installation
+
+```bash
+pip install dvc-databricks
+```
+
+Once installed, the `dbvol://` remote protocol is automatically available to DVC in every process — no imports or additional configuration needed.
+
+---
+
+## Setup
+
+### 1. Initialize DVC in your repository (if not already done)
+
+```bash
+dvc init
+git add .dvc
+git commit -m "initialize DVC"
+```
+
+### 2. Add the Databricks Volume as a DVC remote
+
+```bash
+dvc remote add -d myremote \
+    dbvol:///Volumes/<catalog>/<schema>/<volume>/<path>
+```
+
+Example:
+
+```bash
+dvc remote add -d myremote \
+    dbvol:///Volumes/ml_catalog/datasets/storage/dvc_cache
+```
+
+### 3. Set your Databricks profile
+
+```bash
+export DATABRICKS_CONFIG_PROFILE=<your-profile-name>
+```
+
+> **Note:** DVC remotes do not support arbitrary config keys, so the Databricks
+> profile must be provided via this environment variable — it cannot be stored
+> in `.dvc/config`. Add the export to your `~/.zshrc` or `~/.bashrc` to make
+> it permanent.
+
+---
+
+## Usage
+
+### Track a data file
+
+```bash
+dvc add data/dataset.csv
+```
+
+This creates `data/dataset.csv.dvc` — a small pointer file that goes into git.
+The actual data file must listed in `.gitignore`.
+
+### Push data to the Volume
+
+```bash
+dvc push
+```
+
+Uploads the file to your Databricks Volume via the Databricks SDK.
+
+### Commit the pointer to git
+
+```bash
+git add data/dataset.csv.dvc .gitignore
+git commit -m "track dataset v1 with DVC"
+git push
+```
+
+### Pull data in another environment
+
+```bash
+git clone <your-repo>
+pip install dvc-databricks
+export DATABRICKS_CONFIG_PROFILE=<your-profile-name>
+dvc pull
+```
+
+---
+
+## How it works
+
+```
+Your git repo                   Databricks Volume (S3 / ADLS)
+──────────────────              ───────────────────────────────────
+data/dataset.csv.dvc  ──────►  /Volumes/catalog/schema/vol/
+.dvc/config                     └── files/md5/
+                                    ├── ab/cdef1234...   ← actual data
+                                    └── 9f/123abc...     ← actual data
+```
+
+**`dvc add`** hashes the file and stores it in the local DVC cache (`.dvc/cache`).
+A `.dvc` pointer file containing the MD5 hash is created next to your data file.
+
+**`dvc push`** uploads from the local cache to the Volume using the Databricks
+Files API (`WorkspaceClient.files.upload`). Files are stored content-addressed:
+`<volume_path>/files/md5/<hash[:2]>/<hash[2:]>`.
+
+**`dvc pull`** downloads from the Volume into the local cache, then restores
+the file to its original path.
+
+Only `.dvc` pointer files are ever committed to git — the data stays on the Volume.
+
+---
+
+## Architecture
+
+The plugin follows the same pattern as official DVC plugins (`dvc-gdrive`, `dvc-s3`):
+
+| Class | Base | Role |
+|-------|------|------|
+| `DatabricksVolumesFileSystem` | `dvc_objects.FileSystem` | DVC-facing layer: config, checksum strategy, dependency check |
+| `_DatabricksVolumesFS` | `fsspec.AbstractFileSystem` | I/O layer: all Databricks SDK calls |
+
+A `.pth` file installed into `site-packages` ensures the plugin is loaded at
+Python startup in every process (including DVC CLI subprocesses), without
+requiring any manual imports.
+
+---
+
+## Environment variables
+
+| Variable | Description |
+|----------|-------------|
+| `DATABRICKS_CONFIG_PROFILE` | Databricks CLI profile name from `~/.databrickscfg`. Falls back to the default profile if not set. |
+
+---
+
+## License
+
+[MIT](LICENSE) © Óscar Reyes

dvc_databricks-1.0.3/README.md

@@ -0,0 +1,164 @@
+# dvc-databricks
+
+A [DVC](https://dvc.org) remote storage plugin that enables data versioning on **Databricks Unity Catalog Volumes**.
+
+Store large data files on Databricks Volumes (backed by S3 or ADLS), keep only lightweight `.dvc` pointer files in your git repository, and use standard DVC commands — no custom code required.
+
+```bash
+dvc push   # uploads data to Databricks Volume via Databricks SDK
+dvc pull   # downloads data from Databricks Volume
+```
+
+---
+
+## Why this plugin?
+
+Databricks Unity Catalog Volumes cannot be accessed like a plain S3 bucket — all I/O should go through the **Databricks Files API**. This plugin bridges DVC and the Databricks SDK so you can version and share datasets stored on Volumes without ever leaving the standard DVC workflow.
+
+---
+
+## Requirements
+
+- Python >= 3.10
+- [DVC](https://dvc.org/doc/install) >= 3.0
+- [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/install.html) configured with a profile in `~/.databrickscfg`
+- Access to a Databricks Unity Catalog Volume
+
+---
+
+## Installation
+
+```bash
+pip install dvc-databricks
+```
+
+Once installed, the `dbvol://` remote protocol is automatically available to DVC in every process — no imports or additional configuration needed.
+
+---
+
+## Setup
+
+### 1. Initialize DVC in your repository (if not already done)
+
+```bash
+dvc init
+git add .dvc
+git commit -m "initialize DVC"
+```
+
+### 2. Add the Databricks Volume as a DVC remote
+
+```bash
+dvc remote add -d myremote \
+    dbvol:///Volumes/<catalog>/<schema>/<volume>/<path>
+```
+
+Example:
+
+```bash
+dvc remote add -d myremote \
+    dbvol:///Volumes/ml_catalog/datasets/storage/dvc_cache
+```
+
+### 3. Set your Databricks profile
+
+```bash
+export DATABRICKS_CONFIG_PROFILE=<your-profile-name>
+```
+
+> **Note:** DVC remotes do not support arbitrary config keys, so the Databricks
+> profile must be provided via this environment variable — it cannot be stored
+> in `.dvc/config`. Add the export to your `~/.zshrc` or `~/.bashrc` to make
+> it permanent.
+
+---
+
+## Usage
+
+### Track a data file
+
+```bash
+dvc add data/dataset.csv
+```
+
+This creates `data/dataset.csv.dvc` — a small pointer file that goes into git.
+The actual data file must listed in `.gitignore`.
+
+### Push data to the Volume
+
+```bash
+dvc push
+```
+
+Uploads the file to your Databricks Volume via the Databricks SDK.
+
+### Commit the pointer to git
+
+```bash
+git add data/dataset.csv.dvc .gitignore
+git commit -m "track dataset v1 with DVC"
+git push
+```
+
+### Pull data in another environment
+
+```bash
+git clone <your-repo>
+pip install dvc-databricks
+export DATABRICKS_CONFIG_PROFILE=<your-profile-name>
+dvc pull
+```
+
+---
+
+## How it works
+
+```
+Your git repo                   Databricks Volume (S3 / ADLS)
+──────────────────              ───────────────────────────────────
+data/dataset.csv.dvc  ──────►  /Volumes/catalog/schema/vol/
+.dvc/config                     └── files/md5/
+                                    ├── ab/cdef1234...   ← actual data
+                                    └── 9f/123abc...     ← actual data
+```
+
+**`dvc add`** hashes the file and stores it in the local DVC cache (`.dvc/cache`).
+A `.dvc` pointer file containing the MD5 hash is created next to your data file.
+
+**`dvc push`** uploads from the local cache to the Volume using the Databricks
+Files API (`WorkspaceClient.files.upload`). Files are stored content-addressed:
+`<volume_path>/files/md5/<hash[:2]>/<hash[2:]>`.
+
+**`dvc pull`** downloads from the Volume into the local cache, then restores
+the file to its original path.
+
+Only `.dvc` pointer files are ever committed to git — the data stays on the Volume.
+
+---
+
+## Architecture
+
+The plugin follows the same pattern as official DVC plugins (`dvc-gdrive`, `dvc-s3`):
+
+| Class | Base | Role |
+|-------|------|------|
+| `DatabricksVolumesFileSystem` | `dvc_objects.FileSystem` | DVC-facing layer: config, checksum strategy, dependency check |
+| `_DatabricksVolumesFS` | `fsspec.AbstractFileSystem` | I/O layer: all Databricks SDK calls |
+
+A `.pth` file installed into `site-packages` ensures the plugin is loaded at
+Python startup in every process (including DVC CLI subprocesses), without
+requiring any manual imports.
+
+---
+
+## Environment variables
+
+| Variable | Description |
+|----------|-------------|
+| `DATABRICKS_CONFIG_PROFILE` | Databricks CLI profile name from `~/.databrickscfg`. Falls back to the default profile if not set. |
+
+---
+
+## License
+
+[MIT](LICENSE) © Óscar Reyes

dvc_databricks-1.0.3/pyproject.toml

@@ -0,0 +1,80 @@
+[project]
+name = "dvc-databricks"
+version = "1.0.3"
+description = "DVC remote plugin for Databricks Unity Catalog Volumes"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [{ name = "Óscar Reyes" }]
+keywords = ["dvc", "databricks", "data-versioning", "mlops", "unity-catalog"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.10"
+dependencies = [
+    "databricks-sdk>=0.40.0",
+    "dvc>=3.0",
+    "dvc-objects>=5.0",
+    "fsspec",
+]
+
+[project.urls]
+Homepage = "https://github.com/ogreyesp/dvc-databricks"
+Repository = "https://github.com/ogreyesp/dvc-databricks"
+Issues = "https://github.com/ogreyesp/dvc-databricks/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dvc_databricks"]
+
+# force-include places dvc_databricks_startup.pth at the root of the wheel.
+# pip installs wheel-root files directly into site-packages, so Python will
+# execute "import dvc_databricks" at interpreter startup in every process —
+# including DVC CLI subprocesses — registering the dbvol:// protocol.
+[tool.hatch.build.targets.wheel.force-include]
+"src/dvc_databricks_startup.pth" = "dvc_databricks_startup.pth"
+
+[tool.semantic_release]
+version_toml = ["pyproject.toml:project.version"]
+branch = "main"
+changelog_file = "CHANGELOG.md"
+build_command = "pip install hatchling && python -m hatchling build"
+dist_path = "dist/"
+upload_to_pypi = false
+upload_to_release = false
+commit_message = "chore(release): {version} [skip ci]"
+
+[tool.semantic_release.commit_parser_options]
+allowed_tags = [
+    "feat",
+    "fix",
+    "perf",
+    "refactor",
+    "docs",
+    "chore",
+    "style",
+    "test",
+    "build",
+    "ci",
+]
+minor_tags = ["feat"]
+patch_tags = ["fix", "perf", "refactor"]
+
+[tool.semantic_release.remote]
+type = "github"
+
+[tool.semantic_release.publish]
+dist_glob_patterns = ["dist/*"]
+upload_to_vcs_release = true

dvc_databricks-1.0.3/src/dvc_databricks/__init__.py

@@ -0,0 +1,17 @@
+"""
+dvc-databricks — DVC remote plugin for Databricks Unity Catalog Volumes.
+
+Registers the ``dbvol`` protocol into ``dvc_objects.fs.known_implementations``
+so that DVC can resolve ``dbvol://`` remotes in any process where this
+package is installed.
+
+This registration runs on import. The package uses a .pth file (installed
+into site-packages) to ensure this module is imported at Python startup,
+which makes ``dvc push`` / ``dvc pull`` work from the CLI without any
+manual imports.
+"""
+from dvc_objects.fs import known_implementations
+
+known_implementations["dbvol"] = {
+    "class": "dvc_databricks.filesystem.DatabricksVolumesFileSystem",
+}

dvc_databricks-1.0.3/src/dvc_databricks/filesystem.py

@@ -0,0 +1,546 @@
+"""
+DVC filesystem plugin for Databricks Unity Catalog Volumes.
+
+Architecture:
+
+  DatabricksVolumesFileSystem   ← dvc_objects.FileSystem subclass
+      │                            DVC-facing layer: config parsing,
+      │                            checksum strategy, plugin registration
+      │
+      └── self.fs               ← _DatabricksVolumesFS (fsspec.AbstractFileSystem)
+                                   I/O layer: upload, download, list, delete
+                                   via Databricks SDK Files API
+
+When this package is installed, the ``dvc.plugins`` entry point registers
+``DatabricksVolumesFileSystem`` under the ``dbvol`` protocol. DVC discovers
+it automatically — no imports or manual configuration required.
+
+Users configure the remote once:
+
+    dvc remote add -d myremote dbvol:///Volumes/catalog/schema/volume/path
+    export DATABRICKS_CONFIG_PROFILE=<profile>
+
+Then use standard DVC commands as usual:
+
+    dvc push / dvc pull / dvc status ...
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import threading
+from typing import ClassVar
+
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.config import Config
+from dvc_objects.fs.base import FileSystem
+from fsspec import AbstractFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Inner fsspec filesystem — handles raw I/O via Databricks SDK
+# ---------------------------------------------------------------------------
+
+
+class _DatabricksVolumesFS(AbstractFileSystem):
+    """fsspec filesystem that routes all I/O through the Databricks SDK Files API.
+
+    This is the I/O layer used internally by ``DatabricksVolumesFileSystem``.
+    It is not intended to be used directly by end users.
+
+    Args:
+        profile: Databricks CLI profile name from ``~/.databrickscfg``.
+            When ``None``, the SDK reads ``DATABRICKS_CONFIG_PROFILE`` from
+            the environment, then falls back to the default profile.
+        **storage_options: Additional options forwarded to
+            ``AbstractFileSystem.__init__``.
+    """
+
+    protocol = "dbvol"
+
+    def __init__(self, profile: str | None = None, **storage_options):
+
+        super().__init__(**storage_options)
+
+        resolved = profile or os.environ.get("DATABRICKS_CONFIG_PROFILE")
+        cfg = Config(profile=resolved) if resolved else Config()
+        self._client = WorkspaceClient(config=cfg)
+
+    # ------------------------------------------------------------------
+    # Path helpers
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        """Remove the ``dbvol://`` prefix and ensure a leading slash.
+
+        Args:
+            path: Raw path, possibly prefixed with ``dbvol://``. Also accepts
+                a list of paths, in which case each element is processed.
+
+        Returns:
+            Absolute path string starting with ``/``, or a list of such
+            strings when the input is a list.
+        """
+        if isinstance(path, list):
+            return [cls._strip_protocol(p) for p in path]
+
+        path = super()._strip_protocol(path)
+
+        if not path.startswith("/"):
+            path = "/" + path
+
+        return path
+
+    # ------------------------------------------------------------------
+    # Metadata operations
+    # ------------------------------------------------------------------
+
+    def ls(self, path: str, detail: bool = True, **kwargs):
+        """List a directory or return a single-item list for a file.
+
+        Tries the path as a directory first; falls back to ``info()`` if the
+        directory listing raises an exception (i.e. path is a file).
+
+        Args:
+            path: Absolute Volume path to list.
+            detail: If ``True``, return a list of dicts with keys
+                ``name``, ``type``, ``size``, and ``last_modified``.
+                If ``False``, return a list of path strings.
+            **kwargs: Ignored; present for fsspec compatibility.
+
+        Returns:
+            List of dicts (when ``detail=True``) or list of path strings
+            (when ``detail=False``).
+
+        Raises:
+            FileNotFoundError: If the path does not exist.
+        """
+        path = self._strip_protocol(path)
+
+        try:
+            entries = list(self._client.files.list_directory_contents(path))
+            result = []
+
+            for entry in entries:
+                info = {
+                    "name": entry.path,
+                    "type": "directory" if entry.is_directory else "file",
+                    "size": entry.file_size or 0,
+                    "last_modified": entry.last_modified,
+                }
+                result.append(info if detail else entry.path)
+
+            return result
+
+        except Exception:
+            pass
+
+        # Fall back to a single-file lookup via info() to avoid duplicating
+        # the metadata retrieval logic.
+        info = self.info(path)  # raises FileNotFoundError if path does not exist
+        return [info] if detail else [info["name"]]
+
+    def info(self, path: str, **kwargs) -> dict:
+        """Return metadata for a single file or directory.
+
+        Tries a file metadata lookup first (cheaper), then falls back to a
+        directory metadata lookup.
+
+        Args:
+            path: Absolute Volume path.
+            **kwargs: Ignored; present for fsspec compatibility.
+
+        Returns:
+            Dict with keys ``name`` (str), ``type`` (``"file"`` or
+            ``"directory"``), and ``size`` (int, bytes).
+
+        Raises:
+            FileNotFoundError: If the path does not exist.
+        """
+        path = self._strip_protocol(path)
+
+        try:
+            meta = self._client.files.get_metadata(path)
+
+            return {"name": path, "type": "file", "size": meta.content_length or 0}
+        except Exception:
+            pass
+
+        try:
+            self._client.files.get_directory_metadata(path)
+
+            return {"name": path, "type": "directory", "size": 0}
+        except Exception:
+            raise FileNotFoundError(f"No such file or directory: {path!r}")
+
+    def exists(self, path: str, **kwargs) -> bool:
+        """Return ``True`` if *path* exists on the Volume, ``False`` otherwise.
+
+        Args:
+            path: Absolute Volume path to check.
+            **kwargs: Ignored; present for fsspec compatibility.
+
+        Returns:
+            ``True`` if the path exists, ``False`` if not.
+        """
+        try:
+            self.info(path)
+            return True
+        except FileNotFoundError:
+            return False
+
+    # ------------------------------------------------------------------
+    # Directory operations
+    # ------------------------------------------------------------------
+
+    def mkdir(self, path: str, create_parents: bool = True, **kwargs):
+        """Create a directory on the Volume.
+
+        Args:
+            path: Absolute Volume path for the new directory.
+            create_parents: Ignored — the Databricks Files API always
+                creates intermediate directories automatically.
+            **kwargs: Ignored; present for fsspec compatibility.
+        """
+        path = self._strip_protocol(path)
+        self._client.files.create_directory(path)
+
+    def makedirs(self, path: str, exist_ok: bool = False):
+        """Create a directory and all intermediate parents.
+
+        Args:
+            path: Absolute Volume path for the new directory.
+            exist_ok: If ``False``, re-raises any exception thrown by the
+                API. If ``True``, suppresses those exceptions.
+        """
+        path = self._strip_protocol(path)
+        try:
+            self._client.files.create_directory(path)
+        except Exception:
+            if not exist_ok:
+                raise
+
+    def rm_file(self, path: str):
+        """Delete a single file from the Volume.
+
+        Args:
+            path: Absolute Volume path of the file to delete.
+        """
+        path = self._strip_protocol(path)
+        self._client.files.delete(path)
+
+    def rm(self, path, recursive: bool = False, **kwargs):
+        """Delete one or more files or directories from the Volume.
+
+        Args:
+            path: Absolute Volume path (str) or list of paths to delete.
+            recursive: If ``True``, recursively delete directory contents
+                before deleting the directory itself.
+            **kwargs: Ignored; present for fsspec compatibility.
+        """
+        paths = path if isinstance(path, list) else [path]
+
+        for p in paths:
+            p = self._strip_protocol(p)
+
+            if recursive and self.isdir(p):
+
+                for entry in self.ls(p, detail=True):
+                    self.rm(entry["name"], recursive=True)
+            else:
+                self._client.files.delete(p)
+
+    # ------------------------------------------------------------------
+    # File I/O
+    # ------------------------------------------------------------------
+
+    def _open(self, path: str, mode: str = "rb", **kwargs):
+        """Open a file on the Volume for reading or writing.
+
+        For reads, the file is downloaded eagerly into a ``BytesIO`` buffer.
+        For writes, a ``_WriteBuffer`` is returned which uploads on ``close()``.
+
+        Args:
+            path: Absolute Volume path.
+            mode: ``"rb"`` for reading or ``"wb"`` for writing.
+            **kwargs: Ignored; present for fsspec compatibility.
+
+        Returns:
+            A ``BytesIO`` instance (read mode) or a ``_WriteBuffer``
+            instance (write mode).
+
+        Raises:
+            ValueError: If *mode* is neither ``"rb"`` nor ``"wb"``.
+        """
+        path = self._strip_protocol(path)
+
+        if "r" in mode:
+            response = self._client.files.download(path)
+            return io.BytesIO(response.contents.read())
+
+        if "w" in mode:
+            return _WriteBuffer(self._client, path)
+
+        raise ValueError(f"Unsupported mode: {mode!r}")
+
+    def put_file(self, lpath: str, rpath: str, **kwargs):
+        """Upload a single local file to the Volume.
+
+        Args:
+            lpath: Absolute local filesystem path of the source file.
+            rpath: Absolute Volume path of the destination.
+            **kwargs: Ignored; present for fsspec compatibility.
+        """
+        rpath = self._strip_protocol(rpath)
+
+        with open(lpath, "rb") as fh:
+            self._client.files.upload(rpath, fh, overwrite=True)
+
+    def get_file(self, rpath: str, lpath: str, outfile=None, **kwargs):
+        """Download a single file from the Volume to a local path.
+
+        Args:
+            rpath: Absolute Volume path of the source file.
+            lpath: Absolute local filesystem path of the destination.
+                Intermediate directories are created automatically.
+            outfile: If provided, write the downloaded bytes into this
+                file-like object instead of saving to *lpath*.
+            **kwargs: Ignored; present for fsspec compatibility.
+        """
+        rpath = self._strip_protocol(rpath)
+        response = self._client.files.download(rpath)
+
+        if outfile is not None:
+            outfile.write(response.contents.read())
+        else:
+            os.makedirs(os.path.dirname(os.path.abspath(lpath)), exist_ok=True)
+
+            with open(lpath, "wb") as fh:
+                fh.write(response.contents.read())
+
+
+class _WriteBuffer(io.RawIOBase):
+    """Write-only in-memory buffer that uploads to Databricks on ``close()``.
+
+    fsspec's ``_open(mode="wb")`` contract expects a file-like object.
+    Because the Databricks Files API requires the full content to be
+    available at upload time (it is not a streaming multipart API), we
+    accumulate all ``write()`` calls in a ``BytesIO`` buffer and perform
+    a single ``files.upload()`` call when the buffer is closed.
+
+    The upload is triggered exactly once, either by an explicit ``close()``
+    call or when used as a context manager (``with fs.open(path, "wb") as f``).
+
+    Example:
+        >>> with fs.open("/Volumes/catalog/schema/vol/file.csv", "wb") as f:
+        ...     f.write(b"col1,col2\\n1,2\\n")
+    """
+
+    def __init__(self, client, path: str):
+        """Initialize the buffer.
+
+        Args:
+            client: An authenticated ``WorkspaceClient`` instance.
+            path: Absolute Volume path where the file will be written,
+                e.g. ``/Volumes/catalog/schema/volume/subdir/file.csv``.
+        """
+        self._client = client
+        self._path = path
+        self._buf = io.BytesIO()
+
+    def write(self, data: bytes) -> int:
+        """Append *data* to the in-memory buffer.
+
+        No network call is made at this point.
+
+        Args:
+            data: Bytes to buffer.
+
+        Returns:
+            Number of bytes written.
+        """
+        return self._buf.write(data)
+
+    def close(self):
+        """Flush the buffer to the Databricks Volume and close the stream.
+
+        Performs a single ``files.upload()`` call with the accumulated
+        buffer contents. Subsequent calls are no-ops (guarded by
+        ``self.closed``).
+        """
+        if not self.closed:
+            self._buf.seek(0)
+            self._client.files.upload(self._path, self._buf, overwrite=True)
+
+        super().close()
+
+    def readable(self) -> bool:
+        """Return ``False`` — this stream is write-only.
+
+        Returns:
+            Always ``False``.
+        """
+        return False
+
+    def writable(self) -> bool:
+        """Return ``True`` — this stream accepts ``write()`` calls.
+
+        Returns:
+            Always ``True``.
+        """
+        return True
+
+    def seekable(self) -> bool:
+        """Return ``False`` — seeking is not supported on the public interface.
+
+        The internal ``BytesIO`` buffer is seeked internally by ``close()``
+        before uploading, but callers must not rely on seek support.
+
+        Returns:
+            Always ``False``.
+        """
+        return False
+
+    def __enter__(self):
+        """Return *self* to support usage as a context manager.
+
+        Returns:
+            This ``_WriteBuffer`` instance.
+        """
+        return self
+
+    def __exit__(self, *args):
+        """Close the buffer and trigger the upload on context manager exit.
+
+        Args:
+            *args: Exception info (type, value, traceback) — ignored.
+        """
+        self.close()
+
+
+# ---------------------------------------------------------------------------
+# DVC plugin — dvc_objects.FileSystem wrapper
+# ---------------------------------------------------------------------------
+
+
+class DatabricksVolumesFileSystem(FileSystem):
+    """DVC remote filesystem backed by Databricks Unity Catalog Volumes.
+
+    Extends ``dvc_objects.fs.base.FileSystem``.
+
+    DVC delegates all storage operations to ``self.fs`` (a
+    ``_DatabricksVolumesFS`` instance), which communicates with the
+    Databricks Volume via the SDK Files API — no direct S3 access.
+
+    Configuration (one-time setup per repo):
+
+        dvc remote add -d myremote \\
+            dbvol:///Volumes/catalog/schema/volume/dvc_cache
+        export DATABRICKS_CONFIG_PROFILE=<profile>
+
+    After that, standard DVC commands work without any code changes:
+
+        dvc push / dvc pull / dvc status
+
+    Note:
+        ``DATABRICKS_CONFIG_PROFILE`` must be set in the environment because
+        DVC remotes do not support arbitrary config keys. The profile cannot
+        be stored in ``.dvc/config``.
+    """
+
+    protocol = "dbvol"
+    PARAM_CHECKSUM = "md5"
+    # Format: {"pip_package_name": "importable.module.name"}
+    # dvc_objects calls find_spec() on the value to check the dep is installed.
+    REQUIRES: ClassVar[dict[str, str]] = {"databricks-sdk": "databricks.sdk"}
+
+    def __init__(self, **config):
+        """Parse DVC remote config and prepare the filesystem.
+
+        Args:
+            **config: DVC remote configuration dict. Expected keys:
+
+                - ``url`` (str): Full remote URL, e.g.
+                  ``dbvol:///Volumes/catalog/schema/volume/path``.
+                - ``profile`` (str, optional): Databricks CLI profile name.
+                  Falls back to ``DATABRICKS_CONFIG_PROFILE`` env var.
+        """
+        super().__init__(**config)
+        self.url = config["url"]
+        self._profile = config.get("profile") or os.environ.get(
+            "DATABRICKS_CONFIG_PROFILE"
+        )
+        self._fs_instance: _DatabricksVolumesFS | None = None
+        self._fs_lock = threading.RLock()
+
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath: str) -> dict:
+        """Extract constructor kwargs from a remote URL.
+
+        Called by DVC when parsing ``dvc remote add`` URLs. Returns the
+        URL as-is so it can be forwarded to ``__init__`` as ``config["url"]``.
+
+        Args:
+            urlpath: Full remote URL, e.g.
+                ``dbvol:///Volumes/catalog/schema/volume/path``.
+
+        Returns:
+            Dict with a single ``url`` key.
+        """
+        return {"url": urlpath}
+
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        """Remove the ``dbvol://`` prefix and ensure a leading slash.
+
+        Args:
+            path: Raw path, possibly prefixed with ``dbvol://``.
+
+        Returns:
+            Absolute path string starting with ``/``.
+        """
+        if isinstance(path, list):
+            return [cls._strip_protocol(p) for p in path]
+
+        if path.startswith("dbvol://"):
+            path = path[len("dbvol://") :]
+
+        if not path.startswith("/"):
+            path = "/" + path
+
+        return path
+
+    def unstrip_protocol(self, path: str) -> str:
+        """Reconstruct the full ``dbvol://`` URL from an absolute path.
+
+        Args:
+            path: Absolute Volume path, e.g.
+                ``/Volumes/catalog/schema/volume/file``.
+
+        Returns:
+            Full URL string, e.g.
+            ``dbvol:///Volumes/catalog/schema/volume/file``.
+        """
+        return f"dbvol://{path}"
+
+    @property
+    def fs(self) -> _DatabricksVolumesFS:
+        """Return the underlying fsspec filesystem, created lazily and cached.
+
+        Thread-safe: uses an ``RLock`` to ensure only one instance is created
+        even under concurrent access.
+
+        Returns:
+            A ``_DatabricksVolumesFS`` instance authenticated with the
+            configured Databricks profile.
+        """
+        with self._fs_lock:
+            if self._fs_instance is None:
+                self._fs_instance = _DatabricksVolumesFS(profile=self._profile)
+
+        return self._fs_instance

dvc_databricks-1.0.3/src/dvc_databricks_startup.pth

@@ -0,0 +1 @@
+import dvc_databricks