fsspec-databricks 0.1.9__tar.gz → 0.2.0__tar.gz

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.github/workflows/check.yaml

@@ -19,7 +19,7 @@ jobs:
     if: ${{ inputs.lint }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.github/workflows/on_push.yaml

@@ -38,7 +38,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
           - "3.11"
           - "3.12"
           - "3.13"

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.github/workflows/on_release.yaml

@@ -18,4 +18,4 @@ jobs:
       python-version: "3.14"
     permissions:
       id-token: write
-      contents: read
+      contents: read

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.github/workflows/publish.yaml

@@ -24,10 +24,10 @@ jobs:
       name: ${{ inputs.environment }}
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ inputs.python-version }}
 
@@ -40,4 +40,4 @@ jobs:
         run: uv build
 
       - name: Publish Python package
-        run: uv publish
+        run: uv publish

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.github/workflows/test.yaml

@@ -61,10 +61,10 @@ jobs:
     environment: ${{ inputs.environment }}
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Set up Python ${{ inputs.python-version }}
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ inputs.python-version }}
 
@@ -81,7 +81,7 @@ jobs:
         run: |
           test_path="${{ github.ref_name }}/${{ inputs.python-version }}"
 
-          has_error=no          
+          has_error=no
           if [[ "${{ vars.FSSPEC_DATABRICKS_VOLUME_TEST_ROOT }}" == "" ]]; then
             echo "GitHub Actions variable FSSPEC_DATABRICKS_VOLUME_TEST_ROOT must be set."
             has_error=yes
@@ -126,14 +126,14 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ inputs.coverage && !cancelled() }}
-        uses: codecov/codecov-action@v6
+        uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           report_type: coverage
 
       - name: Upload test results to Codecov
         if: ${{ inputs.coverage && !cancelled() }}
-        uses: codecov/codecov-action@v6
+        uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          report_type: test_results
+          report_type: test_results

{fsspec_databricks-0.1.9 → fsspec_databricks-0.2.0}/.pre-commit-config.yaml

@@ -5,14 +5,14 @@ exclude: >
 repos:
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-json
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.3
+    rev: v0.17.2
     hooks:
       - id: ruff-check
         args: [ --fix, "--show-fixes"]

fsspec_databricks-0.2.0/AUTHENTICATION.md

@@ -0,0 +1,81 @@
+# Authentication
+
+`fsspec-databricks` uses Databricks Unified Authentication, through the Databricks Python SDK. Its parameters and
+environment variables are in the
+[Databricks Python SDK documentation](https://databricks-sdk-py.readthedocs.io/en/latest/authentication.html).
+
+---
+
+## Default
+
+With Databricks Unified Authentication configured, e.g. a default profile, no parameter is needed:
+
+```python
+from fsspec_databricks import DatabricksFileSystem
+
+fs = DatabricksFileSystem()
+```
+
+---
+
+## Constructor parameters
+
+`DatabricksFileSystem` takes the parameters of the SDK's `WorkspaceClient`:
+
+```python
+# Personal access token
+fs = DatabricksFileSystem(host=host_url, token=access_token)
+
+# Another profile
+fs = DatabricksFileSystem(profile="production")
+```
+
+---
+
+## Environment variables
+
+```bash
+export DATABRICKS_CONFIG_PROFILE=production
+```
+
+```python
+fs = DatabricksFileSystem()  # uses the "production" profile
+```
+
+---
+
+## fsspec configuration
+
+[fsspec's configuration](https://filesystem-spec.readthedocs.io/en/latest/features.html#configuration) sets default
+parameters for the `dbfs` protocol:
+
+```python
+import fsspec
+
+fsspec.config.conf["dbfs"] = {"profile": "production"}
+
+fs = DatabricksFileSystem()  # uses the "production" profile
+```
+
+Or with environment variables named `FSSPEC_DBFS_<PARAMETER>`, read when fsspec is imported:
+
+```bash
+export FSSPEC_DBFS_PROFILE=production
+```
+
+---
+
+## `WorkspaceClient`
+
+`DatabricksFileSystem` can use a `WorkspaceClient` of yours, with its authentication:
+
+```python
+from databricks.sdk import WorkspaceClient
+
+client = WorkspaceClient(...)
+
+fs = DatabricksFileSystem(client=client)
+```
+
+A file system created with a `WorkspaceClient` is generally not picklable, as `WorkspaceClient` objects are not. Use
+another way to authenticate where file systems are pickled, e.g. with Dask.

fsspec_databricks-0.2.0/CLAUDE.md

@@ -0,0 +1,108 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+All Python goes through uv: never call `python` or `pip` directly.
+
+```bash
+uv sync                                         # the package, editable, with the dev dependencies
+uv run pytest                                   # all tests: most need a Databricks workspace (see below)
+uv run pytest tests/test_path.py -k volume      # one file / matching tests
+uv run pytest --cov                             # with coverage of src/
+uvx ruff check .                                # lint (ruff is not a dev dependency; config in pyproject.toml)
+uvx ruff format .                               # format
+uvx pre-commit run --all-files                  # the hooks: ruff, whitespace, end of files, YAML
+```
+
+CI, in `.github/workflows/on_push.yaml`, runs on pushes:
+
+- ruff check and ruff format check, with the latest ruff;
+- pytest on Python 3.14 against the default workspace, with coverage on `main`;
+- on `main`, pytest on the other supported Python versions;
+- pytest on Azure, AWS or GCP workspaces named in the `TEST_ENVIRONMENTS` variable.
+
+Releases are published GitHub releases. DEVELOPMENT.md describes them.
+
+## Tests and layout
+
+DEVELOPMENT.md describes the layout, the tests and their setup. Read its Tests section before writing tests. The traps:
+
+- Most tests run against a real workspace. They need credentials and the two test root variables, from `.env` or the
+  environment.
+- The session fixtures `volume_test_root` and `workspace_test_root` delete everything in the test roots first. Never
+  point them at folders holding anything else.
+- Without credentials, run `tests/test_file.py`, `tests/test_volume.py`, `tests/test_path.py` and `tests/test_utils.py`.
+  `test_volume.py` and `test_file.py` use the fake Files API in `tests/dummy_api.py`.
+- A test of anything a workspace serves gets a check without one where possible, on the fake API or with mocks.
+- Test helpers shared by files are in `tests/utils.py`.
+- `tests/conftest.py` sets `cachable = False`, so each test gets new file system instances.
+
+## Architecture
+
+DEVELOPMENT.md's Architecture section describes each part: read it before changing one. The traps:
+
+- `DatabricksFileSystem` routes each call to a backend by path. It holds no file system logic of its own, except for
+  the root folder and transfers between backends.
+- Its methods get the backend and the path to pass with `_resolve(path)`. Inside a workspace, that path is a local
+  path, and names returned are mapped back with `from_local_path`.
+- Base class methods in `base.py` only check preconditions. Subclasses call `super()` first, then the API.
+- Wrap Databricks SDK calls in `error_mapping(path)`, so that callers get `OSError` subclasses.
+- Parse paths with the helpers of `path.py`. Every method takes `dbfs:/` URLs and POSIX paths alike.
+- aiohttp objects are created and used in the Volume backend's IO event loop. `TCPConnector` needs a running loop.
+- Never block on the IO loop from code running in it. File objects' synchronous methods raise there.
+- `DatabricksFileSystem.__init__` takes every parameter of the SDK's `WorkspaceClient`, in its order.
+  `test_dbx_fs_accepts_workspace_client_params` fails when the SDK adds one: add it to the signature, the docstring
+  and `config_params`.
+- File systems must stay picklable. A new attribute holding a client, a loop, a thread or a session is dropped in
+  `__getstate__` and reset in `__setstate__`.
+- Multipart upload uses an undocumented Databricks API. Keep its fallback to resumable upload working.
+- Volume write block sizes are multiples of 256 KB.
+- Workspace files are cached whole in memory, up to the API's 10 MB.
+- Chatty debug logs go under `if self.verbose_debug_log:`.
+
+## Conventions
+
+### Code
+
+- Comments are short, and say why the code does something not obvious. Keep that density and tone.
+- Type hints use built-in generics and `X | None`.
+- Each module ends with `__all__`.
+
+### Docstrings
+
+Docstrings of the public API are in numpy style: `Parameters`, `Returns`, `Raises`. The public API is what users
+construct or call:
+
+- `DatabricksFileSystem` and `use`;
+- `VolumeFileSystem`, `WorkspaceFileSystem` and `DBFS`, with their constructors and `open`.
+
+Internal docstrings stay one line or short prose: file objects, helpers and underscore names.
+
+### Writing docs
+
+Write docs so that people can review them. That covers this file, DEVELOPMENT.md and the user docs.
+
+- One idea per sentence: about 30 words, never more than 50.
+- At most one parenthesis per sentence, never nested.
+- Three or more items make a list.
+- Add a new fact as its own sentence or bullet where it belongs. Never append it to an existing sentence.
+- Name an identifier only when the reader needs it to find the code. A function's details belong in its docstring.
+- Describe what is, not what was: git keeps the history.
+- When editing a passage that breaks these rules, rewrite it.
+
+### Where docs go
+
+- README.md is the quick start: features, installation, paths, compatibility, status and limitations.
+- USAGE.md has the user-facing details: operations, each backend's behaviour, and the options.
+- AUTHENTICATION.md covers the ways to configure authentication.
+- DEVELOPMENT.md has setup, tests, CI, releasing, the design choices and the architecture.
+
+Update the docs when behaviour, parameters or options change. Keep user docs short and compact: what users do, not how
+it works or why.
+
+### Links
+
+README.md, USAGE.md and AUTHENTICATION.md are user docs. README.md uses absolute GitHub URLs for links to files, as
+PyPI shows it but cannot resolve relative paths. The other docs are read on GitHub only, so they use relative paths.