fuse4dbricks 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuse4dbricks-0.7.0/.env.example +47 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/.gitignore +3 -0
- fuse4dbricks-0.7.0/CHANGELOG.md +145 -0
- fuse4dbricks-0.7.0/PKG-INFO +335 -0
- fuse4dbricks-0.7.0/README.md +312 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/api/errors.py +7 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/api/uc_client.py +273 -45
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/auth/provider.py +115 -22
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/auth_manager.py +24 -5
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/data_manager.py +17 -5
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/inode_manager.py +65 -7
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/metadata_manager.py +187 -44
- fuse4dbricks-0.7.0/fuse4dbricks/fs/operations.py +962 -0
- fuse4dbricks-0.7.0/fuse4dbricks/fs/securable_filter.py +75 -0
- fuse4dbricks-0.7.0/fuse4dbricks/fs/write_buffer.py +83 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/main.py +78 -11
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/storage/persistence.py +20 -2
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/pyproject.toml +4 -3
- fuse4dbricks-0.7.0/tests/test_auth_manager.py +341 -0
- fuse4dbricks-0.7.0/tests/test_auth_provider.py +362 -0
- fuse4dbricks-0.7.0/tests/test_data_manager.py +285 -0
- fuse4dbricks-0.7.0/tests/test_e2e_mount.py +409 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/tests/test_inode_manager.py +83 -5
- fuse4dbricks-0.7.0/tests/test_main.py +83 -0
- fuse4dbricks-0.7.0/tests/test_metadata_manager.py +662 -0
- fuse4dbricks-0.7.0/tests/test_operations.py +1672 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/tests/test_persistence.py +19 -1
- fuse4dbricks-0.7.0/tests/test_securable_filter.py +126 -0
- fuse4dbricks-0.7.0/tests/test_uc_client.py +536 -0
- fuse4dbricks-0.7.0/tests/test_uc_client_live.py +258 -0
- fuse4dbricks-0.7.0/tests/test_utils.py +206 -0
- fuse4dbricks-0.7.0/tests/test_write_buffer.py +124 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/uv.lock +548 -500
- fuse4dbricks-0.6.0/CHANGELOG.md +0 -24
- fuse4dbricks-0.6.0/PKG-INFO +0 -171
- fuse4dbricks-0.6.0/README.md +0 -149
- fuse4dbricks-0.6.0/fuse4dbricks/fs/operations.py +0 -416
- fuse4dbricks-0.6.0/fuse4dbricks/mock/__init__,py +0 -0
- fuse4dbricks-0.6.0/fuse4dbricks/mock/pyfuse3.py +0 -2
- fuse4dbricks-0.6.0/tests/test_uc_client.py +0 -169
- fuse4dbricks-0.6.0/tests/test_uc_client_integration.py +0 -73
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/.github/workflows/test.yml +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/__init__.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/api/__init__.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/auth/__init__.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/__init__.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/ram_cache.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/fs/utils.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/fuse4dbricks/storage/__init__.py +0 -0
- {fuse4dbricks-0.6.0 → fuse4dbricks-0.7.0}/tests/test_uc_client_consistency.py +0 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# fuse4dbricks — environment for live / end-to-end tests
|
|
2
|
+
#
|
|
3
|
+
# Copy this file to .env and fill in real values:
|
|
4
|
+
#
|
|
5
|
+
# cp .env.example .env
|
|
6
|
+
# $EDITOR .env
|
|
7
|
+
#
|
|
8
|
+
# .env is gitignored (this .env.example is the only one committed). Load it
|
|
9
|
+
# into your shell before running the tests or mounting, so the values reach
|
|
10
|
+
# both the fuse4dbricks process and the commands that read the mount:
|
|
11
|
+
#
|
|
12
|
+
# set -a; source .env; set +a
|
|
13
|
+
#
|
|
14
|
+
# What it feeds:
|
|
15
|
+
# - DATABRICKS_HOST / DATABRICKS_TOKEN are the standard Databricks variables.
|
|
16
|
+
# With them set, the credential-free unit tests still run AND the live
|
|
17
|
+
# uc_client integration tests stop skipping:
|
|
18
|
+
# pytest tests/test_uc_client_integration.py
|
|
19
|
+
# - The FUSE4DBRICKS_TEST_* variables below target a real volume for the
|
|
20
|
+
# end-to-end mount smoke test (real readdir / getattr / read / EACCES).
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# --- Databricks workspace + credentials -------------------------------------
|
|
24
|
+
|
|
25
|
+
# Workspace URL of your Databricks instance.
|
|
26
|
+
DATABRICKS_HOST="https://adb-xxxxxxxxxxxxxxxx.azuredatabricks.net"
|
|
27
|
+
|
|
28
|
+
# Personal access token (starts with "dapi"). Create one at:
|
|
29
|
+
# $DATABRICKS_HOST/settings/user/developer/access-tokens
|
|
30
|
+
DATABRICKS_TOKEN="dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# --- End-to-end mount test target -------------------------------------------
|
|
34
|
+
|
|
35
|
+
# A Unity Catalog volume you can read, in /Volumes form:
|
|
36
|
+
# /Volumes/<catalog>/<schema>/<volume>
|
|
37
|
+
FUSE4DBRICKS_TEST_VOLUME="/Volumes/my_catalog/my_schema/my_volume"
|
|
38
|
+
|
|
39
|
+
# A known existing file inside that volume, given as a path RELATIVE to
|
|
40
|
+
# FUSE4DBRICKS_TEST_VOLUME (used for stat / cat / multi-chunk read checks).
|
|
41
|
+
# Pick something non-trivial in size if you want to exercise chunk streaming
|
|
42
|
+
# and prefetch (the chunk size is 8 MiB).
|
|
43
|
+
# e.g. "subdir/data.csv"
|
|
44
|
+
FUSE4DBRICKS_TEST_FILE="path/to/file.csv"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# 0.7.0 (2026-06-17)
|
|
2
|
+
|
|
3
|
+
- Fix files written from a Windows application through WSL being uploaded at
|
|
4
|
+
the correct size but filled with zeros. A path-based `setattr(size)` (which
|
|
5
|
+
WSL's 9P layer emits instead of an `ftruncate` carrying the handle) now
|
|
6
|
+
resizes the open writer's buffer instead of reading the not-yet-uploaded
|
|
7
|
+
remote object. A short read while preserving bytes in a true path-based
|
|
8
|
+
truncate now fails with `EIO` instead of silently zero-filling.
|
|
9
|
+
- Add `--single-principal`: use one Databricks identity for every request
|
|
10
|
+
regardless of the requesting uid, resolving the token from the fuse4dbricks
|
|
11
|
+
process's own credentials (its environment and the launching user's
|
|
12
|
+
`~/.databrickscfg`) or from a token written to `.auth`. Intended for
|
|
13
|
+
single-user machines; combine with `--allow-other` so a request carrying an
|
|
14
|
+
undocumented uid (e.g. the Windows file explorer over WSL) is served by the
|
|
15
|
+
one token without fuse4dbricks needing root to read the requesting process.
|
|
16
|
+
- Add `--securable-allowlist` and `--securable-denylist` to restrict which
|
|
17
|
+
Unity Catalog securables are listable and accessible. Each takes a
|
|
18
|
+
comma-separated list of dotted securables (`catalog`, `catalog.schema` or
|
|
19
|
+
`catalog.schema.volume`). A denylist hides the listed securables and
|
|
20
|
+
everything beneath them; an allowlist exposes only the listed securables
|
|
21
|
+
(their parent catalog/schema stay navigable). Deny wins when both are given.
|
|
22
|
+
This is a mount-wide visibility filter, not a per-user boundary, and does not
|
|
23
|
+
replace Unity Catalog's own permission checks.
|
|
24
|
+
- Implement `rename` and `setattr`. Files can be moved/renamed (via
|
|
25
|
+
copy-then-delete, since the Files API has no server-side move) and truncated;
|
|
26
|
+
directory rename returns `EXDEV` so `mv` falls back to a recursive copy.
|
|
27
|
+
`chmod`/`chown`/`utimes` are accepted as no-ops (Unity Catalog has no POSIX
|
|
28
|
+
metadata) so tools such as `tar` and `cp -p` don't fail.
|
|
29
|
+
- Implement `statfs` so `df` works and tools that pre-check free space before
|
|
30
|
+
writing succeed. A large synthetic capacity is reported (Unity Catalog
|
|
31
|
+
exposes no quota), with zero availability on a `--read-only` mount.
|
|
32
|
+
- Fix silent data loss on writes: opening a file `O_WRONLY` without `O_TRUNC`
|
|
33
|
+
now pre-loads the existing content so untouched bytes are preserved, and
|
|
34
|
+
small buffered writes are flushed before upload (they could previously be
|
|
35
|
+
uploaded as zero bytes).
|
|
36
|
+
- Security: validate that the default `~/.databrickscfg` is a regular file
|
|
37
|
+
owned by the requesting uid, the same way the env-supplied config path is
|
|
38
|
+
already checked. Prevents a symlink planted by one user (under `--allow-other`
|
|
39
|
+
+ root) from making root read and cache another user's token.
|
|
40
|
+
- Fix `create` not incrementing the kernel lookup count: it returns an entry
|
|
41
|
+
the kernel will later `forget`, so the new inode could be freed while still
|
|
42
|
+
referenced.
|
|
43
|
+
- Map a 412 (the file changed under an `If-Unmodified-Since` read) to `ESTALE`
|
|
44
|
+
instead of `EIO`, so a mid-read modification is reported accurately.
|
|
45
|
+
- Honor the HTTP-date form of the `Retry-After` header (previously only the
|
|
46
|
+
integer-seconds form was used).
|
|
47
|
+
- Refresh the access token on a 401 during an SDK file upload; the upload path
|
|
48
|
+
bypassed the request-layer 401 retry and so failed permanently on an expired
|
|
49
|
+
token.
|
|
50
|
+
- Don't let an inode-side `st_size` mutation (from a write or truncate) corrupt
|
|
51
|
+
the shared metadata cache: `add_entry` now stores a copy of the attributes.
|
|
52
|
+
- A coalescing `lookup_child` follower that wakes to a cache miss now surfaces a
|
|
53
|
+
retryable `EAGAIN` instead of running its own lookup and notifying a coalescer
|
|
54
|
+
key it does not own.
|
|
55
|
+
- Performance: stop running a blocking `os.makedirs` on every chunk read; cache
|
|
56
|
+
shard directories are created only on the write path, off the event loop.
|
|
57
|
+
- Security: A user could use DATABRICKS_CONFIG_PROFILE to make fuse4dbricks read any user profile if it ran as root.
|
|
58
|
+
- Retry transient API failures (429 rate limits, 5xx server errors and
|
|
59
|
+
connection errors) with exponential backoff and jitter, honoring the
|
|
60
|
+
`Retry-After` header on 429 responses.
|
|
61
|
+
- Robustness: a failed chunk download no longer escapes the download worker.
|
|
62
|
+
Previously the exception propagated to the trio nursery and unmounted the
|
|
63
|
+
whole filesystem for every user; now the worker logs and survives, and the
|
|
64
|
+
affected read fails in isolation with `EIO`.
|
|
65
|
+
- Fix `getattr` swallowing `ENOENT` (deleted files) and other domain errors
|
|
66
|
+
into `EIO`; already-mapped errnos now propagate and `getattr` updates the
|
|
67
|
+
inode attributes in place so replies reflect the current size/mtime.
|
|
68
|
+
- Fix writes to read-only Unity Catalog paths returning `EIO` instead of
|
|
69
|
+
`EACCES` (the catch-all `except` was masking the intended errno). `read`
|
|
70
|
+
likewise preserves already-mapped errnos.
|
|
71
|
+
- Fix `lookup` not incrementing the kernel lookup count, which could let
|
|
72
|
+
`forget` free a still-referenced inode.
|
|
73
|
+
- Refactor: the auth virtual entries overlaid at the root (`.auth`,
|
|
74
|
+
`README.txt`) are now defined by `AuthManager.root_overlay()` instead of
|
|
75
|
+
being hardcoded in `readdir`, removing the names/offset coupling. Dead,
|
|
76
|
+
type-confused root branch in `AuthManager.list_directory` removed.
|
|
77
|
+
- Security: don't turn transient authorization failures into a permanent
|
|
78
|
+
`EACCES`. A rate limit or server outage during a permission check (and a
|
|
79
|
+
coalescing follower that woke to no cached decision) previously looked like
|
|
80
|
+
"permission denied"; transient failures now surface a retryable `EAGAIN`,
|
|
81
|
+
reserving `EACCES` for genuine denials.
|
|
82
|
+
- Re-evaluate cached identity when a user's token changes: writing a new token
|
|
83
|
+
refreshes the cached principal and drops that user's cached permission grants
|
|
84
|
+
only if the principal actually changed.
|
|
85
|
+
- Cache catalog/schema permission grants with the longer catalog TTL (volume
|
|
86
|
+
grants keep the shorter TTL for faster revocation).
|
|
87
|
+
- Key the permission cache by principal (not uid) and drop the cached principal
|
|
88
|
+
whenever a token is invalidated (auth-file write or 401), so a uid switching
|
|
89
|
+
tokens to a different principal can never read the previous principal's
|
|
90
|
+
cached grants.
|
|
91
|
+
- Add a short-lived, per-principal negative (not-found) cache so repeated
|
|
92
|
+
lookups of missing paths are answered without re-hitting Unity Catalog, and a
|
|
93
|
+
coalescing follower can tell "genuinely absent" (ENOENT) from "the lookup
|
|
94
|
+
failed" (retryable). TTL via `--metadata-cache-ttl-negative-sec` (5s default).
|
|
95
|
+
- Surface a retryable `EAGAIN` (instead of a permanent `EACCES`) when identity
|
|
96
|
+
resolution fails transiently and a coalescing follower wakes to no cached
|
|
97
|
+
principal.
|
|
98
|
+
- Clear the listing principal's stale negative entries when a directory is
|
|
99
|
+
listed, so a freshly-created file shown by `ls` is no longer reported as
|
|
100
|
+
missing by a subsequent `stat` until the negative TTL expires.
|
|
101
|
+
- Fix a `lookup` race where a `forget` arriving during the `getattr` await
|
|
102
|
+
could free a still-referenced inode (spurious `ENOENT`); the inode is now
|
|
103
|
+
pinned before the await.
|
|
104
|
+
- Robustness: read `/proc/<pid>/environ` as bytes so a single non-UTF-8
|
|
105
|
+
variable no longer discards the requesting process's whole environment.
|
|
106
|
+
- CLI: replace the inert `--unified-auth` flag (it defaulted to on and could
|
|
107
|
+
never be turned off) with `--no-unified-auth`.
|
|
108
|
+
- Raise a clear error for an unknown auth inode instead of an
|
|
109
|
+
`UnboundLocalError`; drop a stray `print(traceback)` in the directory-listing
|
|
110
|
+
error path in favor of `logger.exception`.
|
|
111
|
+
- Document known limitations: the attribute cache is global (names, sizes and
|
|
112
|
+
modification times may leak between users, but never file contents), users
|
|
113
|
+
map to tokens 1-to-1, and reported file owner/group (`st_uid`/`st_gid`) are
|
|
114
|
+
meaningless.
|
|
115
|
+
- Improve test coverage
|
|
116
|
+
|
|
117
|
+
# 0.6.1 (2026-05-07)
|
|
118
|
+
|
|
119
|
+
- Fix type issue
|
|
120
|
+
- Depend on pyfuse3.4.2 to make sure 3.4.1 is not used (it's yanked)
|
|
121
|
+
|
|
122
|
+
# 0.6.0 (2026-05-07)
|
|
123
|
+
|
|
124
|
+
- Allow setting long access tokens (when token splits in more than one
|
|
125
|
+
write syscall).
|
|
126
|
+
- Add Databricks Unified Auth support (Needs root permissions if running multiuser).
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# 0.5.3 (2026-02-24)
|
|
130
|
+
|
|
131
|
+
- Improved documentation
|
|
132
|
+
- Fixed tests
|
|
133
|
+
|
|
134
|
+
# 0.5.2 (2026-02-24)
|
|
135
|
+
|
|
136
|
+
- Improved error handling.
|
|
137
|
+
|
|
138
|
+
# 0.5.1 (2026-02-23)
|
|
139
|
+
|
|
140
|
+
- Extended README.
|
|
141
|
+
- Better handling closing of running services.
|
|
142
|
+
|
|
143
|
+
# 0.5.0 (2026-02-20)
|
|
144
|
+
|
|
145
|
+
- Initial release
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fuse4dbricks
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: FUSE driver for Databricks Unity Catalog Volumes.
|
|
5
|
+
Project-URL: Homepage, https://github.com/zeehio/fuse4dbricks
|
|
6
|
+
Author-email: Sergio Oller <sergioller@gmail.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Keywords: async,databricks,filesystem,fuse,trio,unity-catalog
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: System :: Filesystems
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: databricks-sdk>=0.20.0
|
|
18
|
+
Requires-Dist: httpx>=0.24.0
|
|
19
|
+
Requires-Dist: platformdirs; sys_platform != 'win32'
|
|
20
|
+
Requires-Dist: pyfuse3>=3.4.2
|
|
21
|
+
Requires-Dist: trio>=0.22.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# fuse4dbricks
|
|
25
|
+
|
|
26
|
+
A filesystem in userspace for mounting the Unity Catalog from Databricks.
|
|
27
|
+
|
|
28
|
+
## Disclaimer
|
|
29
|
+
|
|
30
|
+
This is not an official databricks package. I, the author of this package, am not affiliated to Databricks. My capacity to support this package is very limited or none. I may review issues and pull requests but I won't commit to timelines or features.
|
|
31
|
+
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
This filesystem uses the [public databricks API](https://docs.databricks.com/api/azure/workspace/introduction) to retrieve files, directories and access permissions from the Unity Catalog.
|
|
35
|
+
|
|
36
|
+
**Read and write** access is supported for Unity Catalog Volumes. Users with `WRITE_VOLUME` privilege can create, overwrite and delete files and directories, and can `mv`/rename and truncate files. Files above 5 GB are uploaded via multipart upload handled transparently by the Databricks SDK. Pass `--read-only` to disable all writes to Unity Catalog (useful when mounting a shared volume that should not be modified). `df` reports a (synthetic) capacity so tools that check free space before writing work.
|
|
37
|
+
|
|
38
|
+
**Restricting visibility** to a subset of securables is supported with `--securable-allowlist` and `--securable-denylist`. Each takes a comma-separated list of securables, written dotted: a catalog (`mycatalog`), a schema (`mycatalog.myschema`) or a volume (`mycatalog.myschema.myvol`). With a denylist, the listed securables (and everything beneath them) are hidden and inaccessible. With an allowlist, only the listed securables are listable and accessible — their parent catalog/schema stay navigable so you can reach them. If both are given, only allowlisted securables that are not also denied are accessible (deny wins). This is a mount-wide visibility filter layered on top of — not a replacement for — Unity Catalog's own per-user permission checks.
|
|
39
|
+
|
|
40
|
+
To mitigate latency and improve **performance**, file metadata is cached in-memory. Data is cached
|
|
41
|
+
to a local cache directory (`--disk-cache-dir`) and partially to RAM as well. Options to control
|
|
42
|
+
the sizes of those caches are available.
|
|
43
|
+
|
|
44
|
+
**Credentials** are stored in RAM while the filesystem is mounted.
|
|
45
|
+
|
|
46
|
+
`DATABRICKS_TOKEN`, `DATABRICKS_CONFIG_PROFILE` and `DATABRICKS_CONFIG_FILE` are used to find the databricks configuration file and obtain a valid access token. Alternatively, credentials can be passed by writing a
|
|
47
|
+
personal access token to a virtual file:
|
|
48
|
+
|
|
49
|
+
echo "dapi0000000-2" > /Volumes/.auth/personal_access_token
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
If fuse (`/etc/fuse.conf`) has `user_allow_other` activated, this driver supports the `--allow-other`,
|
|
53
|
+
option so **multiple users** can access it. In this case, the fuse4dbricks process should run from a root account, who should have exclusive access to `--disk-cache-dir`. fuse4dbricks will inspect the environment variables of the requesting process, as well as the requesting user, in order to find the access credentials. This may require reading the `.databrickscfg` file from the requesting user. The cache is shared among all users in this scenario.
|
|
54
|
+
|
|
55
|
+
On a **single-user machine** you can instead pass `--single-principal`, so one Databricks identity serves every request regardless of the requesting uid. The token is resolved from the fuse4dbricks process's own credentials (its environment and the launching user's `~/.databrickscfg`) or from a token written to `.auth` — never from the requesting process. This is especially useful under WSL: combine `--single-principal` with `--allow-other` so the Windows file explorer (whose requests arrive with an undocumented uid) can browse the mount using that single token, without fuse4dbricks needing root to inspect each requesting process.
|
|
56
|
+
|
|
57
|
+
When an access token is missing, revoked or expired, the unity catalog is not accessible anymore and only
|
|
58
|
+
a virtual `/Volumes/README.txt` file appears, with instructions on how to add the access token manually.
|
|
59
|
+
|
|
60
|
+
## Requirements
|
|
61
|
+
|
|
62
|
+
**This package depends on pyfuse3, that requires the `libfuse3-dev` or `fuse3-devel` system package
|
|
63
|
+
being installed. The package name depends on your specific linux distribution.**
|
|
64
|
+
|
|
65
|
+
- Debian / Ubuntu: `sudo apt install libfuse3-dev`
|
|
66
|
+
- RedHat / Fedora: `sudo dnf install fuse3-devel`
|
|
67
|
+
- SUSE: `sudo zypper install fuse3-devel`
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
## Quickstart
|
|
71
|
+
|
|
72
|
+
If you want to see databricks volumes as if you were on Databricks (in /Volumes) you will need admin
|
|
73
|
+
privileges to create /Volumes directory on your computer. Otherwise you can use $HOME/Volumes or whatever
|
|
74
|
+
directory you have access to.
|
|
75
|
+
|
|
76
|
+
pip install "fuse4dbricks"
|
|
77
|
+
mkdir "$HOME/Volumes" # or any other directory
|
|
78
|
+
fuse4dbricks --workspace "https://adb-xxxx.azuredatabricks.net" $HOME/Volumes
|
|
79
|
+
ls $HOME/Volumes
|
|
80
|
+
# Your catalogs will appear
|
|
81
|
+
|
|
82
|
+
This uses databricks unified authentication (e.g. searches for `$HOME/.databrickscfg` in your home). If no unified auth file
|
|
83
|
+
is found, you will find a README file with instructions on how to provide a personal access token.
|
|
84
|
+
|
|
85
|
+
If you are the only user (e.g. on your laptop) and you have admin permissions, feel free to use `/Volumes` so paths are like on databricks:
|
|
86
|
+
|
|
87
|
+
pip install "fuse4dbricks"
|
|
88
|
+
sudo mkdir /Volumes"
|
|
89
|
+
sudo chown $USER /Volumes
|
|
90
|
+
fuse4dbricks --workspace "https://adb-xxxx.azuredatabricks.net" /Volumes
|
|
91
|
+
ls /Volumes
|
|
92
|
+
# Your catalogs will appear on /Volumes
|
|
93
|
+
|
|
94
|
+
## Quick start on WSL
|
|
95
|
+
|
|
96
|
+
This section covers mounting Databricks Unity Catalog volumes under
|
|
97
|
+
[Windows Subsystem for Linux (WSL)](https://learn.microsoft.com/en-us/windows/wsl/) and
|
|
98
|
+
browsing them directly from Windows Explorer.
|
|
99
|
+
|
|
100
|
+
### 1. Install system dependencies
|
|
101
|
+
|
|
102
|
+
On a WSL shell:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
sudo apt update
|
|
106
|
+
sudo apt install -y fuse3 libfuse3-dev
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### 2. Install fuse4dbricks
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install "fuse4dbricks"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### 3. Enable `user_allow_other` in FUSE
|
|
116
|
+
|
|
117
|
+
Edit `/etc/fuse.conf` and uncomment the `user_allow_other` line:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
sudo nano /etc/fuse.conf
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Find the line `#user_allow_other` and remove the leading `#` so it reads:
|
|
124
|
+
|
|
125
|
+
```
|
|
126
|
+
user_allow_other
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Save and close the file.
|
|
130
|
+
|
|
131
|
+
### 4. Set your Databricks credentials
|
|
132
|
+
|
|
133
|
+
Export your workspace URL and personal access token as environment variables:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
export DATABRICKS_HOST="https://adb-xxxx.azuredatabricks.net"
|
|
137
|
+
export DATABRICKS_TOKEN="dapi0000000-2"
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Replace the values with your actual workspace URL and
|
|
141
|
+
[personal access token](https://docs.databricks.com/en/dev-tools/auth/pat.html).
|
|
142
|
+
|
|
143
|
+
### 5. Create the `/Volumes` mount point
|
|
144
|
+
|
|
145
|
+
Create the directory and give your user write access:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
sudo mkdir -p /Volumes
|
|
149
|
+
sudo chown "$USER" /Volumes
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### 6. Mount the filesystem
|
|
153
|
+
|
|
154
|
+
Run `fuse4dbricks` with `--single-principal` (your token is the only one in use) and
|
|
155
|
+
`--allow-other` (required for Windows Explorer to access the mount through WSL):
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
fuse4dbricks \
|
|
159
|
+
--workspace "$DATABRICKS_HOST" \
|
|
160
|
+
--single-principal \
|
|
161
|
+
--allow-other \
|
|
162
|
+
/Volumes
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
**Leave this terminal open.** The mount stays active as long as this process is running.
|
|
166
|
+
|
|
167
|
+
### 7. Verify the mount
|
|
168
|
+
|
|
169
|
+
Open a **new terminal** and check that your catalogs are visible:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
ls /Volumes
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
You should see your Unity Catalog catalogs listed as directories. You can navigate into
|
|
176
|
+
them as you would any local filesystem:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
ls /Volumes/<catalog>/<schema>/
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### 8. Browse from Windows Explorer
|
|
183
|
+
|
|
184
|
+
WSL mounts are accessible from Windows as a network drive. In Windows Explorer:
|
|
185
|
+
|
|
186
|
+
1. Open **Windows Explorer** and look for **Linux** in the left-hand navigation panel
|
|
187
|
+
(under "Network" or directly in the sidebar).
|
|
188
|
+
2. Navigate to your WSL distribution (e.g. `\\wsl.localhost\Ubuntu`).
|
|
189
|
+
3. Browse to the `Volumes` directory — your Databricks catalogs, schemas, and files
|
|
190
|
+
will appear just like any other folder.
|
|
191
|
+
|
|
192
|
+
> **Tip:** You can also pin `\\wsl.localhost\Ubuntu\Volumes` to Quick Access or map it
|
|
193
|
+
> as a network drive for faster access in the future.
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
## Multi user setup
|
|
198
|
+
|
|
199
|
+
If you are on a shared computer (e.g. an HPC), fuse4dbricks can run as a root service and use the access tokens from any user who
|
|
200
|
+
aims to access unity catalog volumes.
|
|
201
|
+
|
|
202
|
+
The databricks token used will depend on the user who asks for the file.
|
|
203
|
+
|
|
204
|
+
A cache is shared so if many users read the same files the access speed is faster. Users can't access other users files, because each
|
|
205
|
+
user has to provide its own access token.
|
|
206
|
+
|
|
207
|
+
- Create a virtual environment and install fuse4dbricks there:
|
|
208
|
+
|
|
209
|
+
# Note that fuse4dbricks requires python>=3.11
|
|
210
|
+
sudo bash
|
|
211
|
+
mkdir /opt/fuse4dbricks
|
|
212
|
+
chmod 755 /opt/fuse4dbricks
|
|
213
|
+
python3.11 -m venv /opt/fuse4dbricks/venv
|
|
214
|
+
source /opt/fuse4dbricks/venv/bin/activate
|
|
215
|
+
python3 -m pip install fuse4dbricks
|
|
216
|
+
deactivate
|
|
217
|
+
exit # exit from sudo bash
|
|
218
|
+
|
|
219
|
+
- Create the mount directory:
|
|
220
|
+
|
|
221
|
+
sudo mkdir /Volumes
|
|
222
|
+
sudo chmod 0755 /Volumes
|
|
223
|
+
|
|
224
|
+
- Create the cache directory:
|
|
225
|
+
|
|
226
|
+
sudo mkdir /var/cache/fuse4dbricks
|
|
227
|
+
sudo chmod 0700 /var/cache/fuse4dbricks
|
|
228
|
+
|
|
229
|
+
- Enable `user_allow_other` support in `/etc/fuse.conf`
|
|
230
|
+
|
|
231
|
+
Edit `/etc/fuse.conf` and uncomment the `user_allow_other` line.
|
|
232
|
+
|
|
233
|
+
- Create a starting script and make it executable:
|
|
234
|
+
|
|
235
|
+
Please replace whatever you need here
|
|
236
|
+
|
|
237
|
+
cat << EOF | sudo tee /opt/fuse4dbricks/fuse4dbricks_start.sh
|
|
238
|
+
#!/bin/bash
|
|
239
|
+
|
|
240
|
+
source /opt/fuse4dbricks/venv/bin/activate
|
|
241
|
+
fuse4dbricks \
|
|
242
|
+
--workspace "https://adb-xxxx.azuredatabricks.net" \
|
|
243
|
+
--disk-cache-dir /var/cache/fuse4dbricks \
|
|
244
|
+
--allow-other \
|
|
245
|
+
--ram-cache-mb 512 \
|
|
246
|
+
--disk-cache-gb 1024 \
|
|
247
|
+
--disk-cache-max-days 30 \
|
|
248
|
+
/Volumes
|
|
249
|
+
EOF
|
|
250
|
+
sudo chmod +x /opt/fuse4dbricks/fuse4dbricks_start.sh
|
|
251
|
+
|
|
252
|
+
- Create a systemd unit
|
|
253
|
+
|
|
254
|
+
cat << EOF | sudo tee /etc/systemd/system/fuse4dbricks.service
|
|
255
|
+
[Unit]
|
|
256
|
+
Description=fuse4dbricks
|
|
257
|
+
After=network.target
|
|
258
|
+
|
|
259
|
+
[Service]
|
|
260
|
+
Type=simple
|
|
261
|
+
User=root
|
|
262
|
+
WorkingDirectory=/opt/fuse4dbricks
|
|
263
|
+
ExecStart=/opt/fuse4dbricks/fuse4dbricks_start.sh
|
|
264
|
+
Restart=on-failure
|
|
265
|
+
RestartSec=5
|
|
266
|
+
|
|
267
|
+
[Install]
|
|
268
|
+
WantedBy=multi-user.target
|
|
269
|
+
EOF
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
- Reload the daemon lists
|
|
273
|
+
|
|
274
|
+
sudo systemctl daemon-reload
|
|
275
|
+
|
|
276
|
+
- Enable and start the service
|
|
277
|
+
|
|
278
|
+
sudo systemctl enable fuse4dbricks
|
|
279
|
+
sudo systemctl start fuse4dbricks
|
|
280
|
+
|
|
281
|
+
## Known limitations
|
|
282
|
+
|
|
283
|
+
These are intentional design trade-offs in the current implementation. We are open to
|
|
284
|
+
discussing them, so feel free to open an issue if any of them is a problem for your use case.
|
|
285
|
+
|
|
286
|
+
- **The attribute cache is global.** File and directory metadata (names, sizes and
|
|
287
|
+
modification times) is cached in memory and shared across all users of a single mount.
|
|
288
|
+
As a consequence, names, sizes and modification times of files may leak between users,
|
|
289
|
+
even between users who do not have access to read them. **File contents never leak**:
|
|
290
|
+
reading data always goes through a per-user permission check against Unity Catalog using
|
|
291
|
+
that user's own token. The metadata cache is shared for performance reasons, to avoid
|
|
292
|
+
repeating Unity Catalog API calls for objects that several users access.
|
|
293
|
+
|
|
294
|
+
- **Users are mapped to tokens 1-to-1.** fuse4dbricks associates a single access token with
|
|
295
|
+
each user (uid). It is therefore not possible for two processes belonging to the same user
|
|
296
|
+
to access different Unity Catalog principals at the same time (for example by setting a
|
|
297
|
+
different `DATABRICKS_CONFIG_PROFILE` per process). The first token resolved for a user is
|
|
298
|
+
reused for all of that user's processes.
|
|
299
|
+
|
|
300
|
+
- **File owner and group are meaningless.** The user and group reported for a file or
|
|
301
|
+
directory (`st_uid`/`st_gid`, what `ls -l` shows in the owner and group columns) are simply
|
|
302
|
+
whoever happened to populate the shared metadata cache for that entry first, so the same
|
|
303
|
+
file may appear owned by different users at different times. These values are irrelevant:
|
|
304
|
+
access is never decided from POSIX ownership, only from a per-user Unity Catalog permission
|
|
305
|
+
check. Do not rely on the reported owner or group for anything.
|
|
306
|
+
|
|
307
|
+
- **Not-found results are briefly cached.** When a path is looked up and does not exist, that
|
|
308
|
+
"not found" answer is cached per user for a short time (`--metadata-cache-ttl-negative-sec`,
|
|
309
|
+
5 seconds by default) to avoid repeatedly asking Unity Catalog about missing paths. As a
|
|
310
|
+
result, a file created (or that becomes visible to you) just after you looked for it may not
|
|
311
|
+
appear until this short TTL expires. Unlike file metadata, these negative results are *not*
|
|
312
|
+
shared between users.
|
|
313
|
+
|
|
314
|
+
- **Writes are object-store semantics, not POSIX semantics.** The Databricks Files API is a
|
|
315
|
+
full-replace store: a file is not visible to other processes until the writing process closes
|
|
316
|
+
it (`release`). Concurrent writers follow last-write-wins — there is no locking. Deleting a
|
|
317
|
+
file while another process has it open will cause that process to receive an I/O error on its
|
|
318
|
+
next network fetch. These are intentional trade-offs for performance on object storage.
|
|
319
|
+
|
|
320
|
+
- **Opening a large file O_RDWR without O_TRUNC downloads the entire file first.** Because
|
|
321
|
+
writes are buffered locally and uploaded on close, the full existing content must be
|
|
322
|
+
downloaded into a temporary file before the open returns. For very large files (multi-GB)
|
|
323
|
+
this can be slow. Use `O_WRONLY` or `O_RDWR | O_TRUNC` when you intend to overwrite the
|
|
324
|
+
file completely.
|
|
325
|
+
|
|
326
|
+
- **Renaming a file copies it.** The Databricks Files API has no server-side move, so renaming
|
|
327
|
+
a file downloads it and re-uploads it under the new name, then deletes the original. This is
|
|
328
|
+
cheap for the write-temp-then-rename pattern editors use, but expensive for large files.
|
|
329
|
+
Renaming a *directory* is not supported and returns `EXDEV`, which makes `mv` fall back to a
|
|
330
|
+
recursive copy.
|
|
331
|
+
|
|
332
|
+
- **Securable allow/deny is a mount-wide visibility filter, not a per-user boundary.**
|
|
333
|
+
`--securable-allowlist`/`--securable-denylist` hide securables for everyone using the mount;
|
|
334
|
+
they are not per-user and do not replace Unity Catalog permissions. A user still needs the
|
|
335
|
+
relevant Unity Catalog privileges to read data that the filter allows.
|