fuse4dbricks 0.7.4__tar.gz → 0.7.5__tar.gz

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 0.7.5 (2026-08-06)
+
+- Fix fatal startup/runtime errors exiting with status 0 instead of 1.
+- Fix a race in Databricks config file validation where ownership was
+  checked on a path and the file was read from that same path separately.
+  Ownership is now checked on the open file descriptor that is read.
+- Fix prefetch reading already-downloaded chunks back from disk on every
+  read, instead of just checking they exist. Reading a large file
+  sequentially could pull roughly an order of magnitude more bytes off
+  local disk than the file actually contains.
+
 # 0.7.4 (2026-06-20)
 
 - Make writes durable and immediately visible on `close()`. The upload ran only

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fuse4dbricks
-Version: 0.7.4
+Version: 0.7.5
 Summary: FUSE driver for Databricks Unity Catalog Volumes.
 Project-URL: Homepage, https://github.com/zeehio/fuse4dbricks
 Author-email: Sergio Oller <sergioller@gmail.com>
@@ -322,6 +322,12 @@ discussing them, so feel free to open an issue if any of them is a problem for y
   appear until this short TTL expires. Unlike file metadata, these negative results are *not*
   shared between users.
 
+- **Revoked permissions can stay effective for a short time.** Catalog and
+  schema access permissions are cached using `--metadata-cache-ttl-catalog-sec`,
+  set to 10 minutes (600 seconds) by default. Volume access permissions are
+  cached using `--metadata-cache-ttl-sec`, set to 30 seconds by default. This
+  reduces the request load on the Unity Catalog API, improving performance.
+
 - **Writes are object-store semantics, not POSIX semantics.** The Databricks Files API is a
   full-replace store: a file is not visible to other processes until the writing process closes
   it (`release`). Concurrent writers follow last-write-wins — there is no locking. Deleting a

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/README.md

@@ -299,6 +299,12 @@ discussing them, so feel free to open an issue if any of them is a problem for y
   appear until this short TTL expires. Unlike file metadata, these negative results are *not*
   shared between users.
 
+- **Revoked permissions can stay effective for a short time.** Catalog and
+  schema access permissions are cached using `--metadata-cache-ttl-catalog-sec`,
+  set to 10 minutes (600 seconds) by default. Volume access permissions are
+  cached using `--metadata-cache-ttl-sec`, set to 30 seconds by default. This
+  reduces the request load on the Unity Catalog API, improving performance.
+
 - **Writes are object-store semantics, not POSIX semantics.** The Databricks Files API is a
   full-replace store: a file is not visible to other processes until the writing process closes
   it (`release`). Concurrent writers follow last-write-wins — there is no locking. Deleting a

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/auth/provider.py

@@ -156,114 +156,111 @@ class DatabricksUnifiedAuthProvider:
             logger.error("Failed to read environ file %s for pid %s. Exception: %s", environ_file, pid, exc)
             return None
 
-    def _is_safe_config_file(self, config_file: str, uid: int) -> bool:
-        """Return True only if ``config_file`` is safe to read on behalf of ``uid``.
-
-        When fuse4dbricks runs as root (the allow_other deployment), it reads
-        config files out of arbitrary users' home directories. A user who
-        controls their own ``~/.databrickscfg`` could symlink it at another
-        user's config file; root would follow the link and cache the *victim's*
-        token under the *attacker's* uid. ``os.stat`` follows symlinks, so the
-        ``st_uid == uid`` check is what defeats that: a symlinked-to victim file
-        is owned by the victim, not the requesting uid, and is rejected. This is
-        applied to *every* config file path, env-supplied or default, never just
-        one branch.
+    def _read_token_from_path(
+        self, config_file: str, uid: int, profile: str
+    ) -> tuple[bool, Optional[str]]:
+        """Reads ``profile``'s token from ``config_file`` on behalf of ``uid``.
+
+        Returns ``(True, token)`` if ``config_file`` exists and passes the
+        ownership/type checks below (``token`` is ``None`` if the profile or
+        its token key is missing -- a content error, not a trust failure, so
+        it is not retried against another location). Returns ``(False,
+        None)`` if the path does not exist or fails validation, so the
+        caller knows to fall back to the next candidate location.
+
+        fuse4dbricks may run as root and read config files out of arbitrary
+        users' home directories (the allow_other deployment), so a path is
+        trusted only if it is owned by ``uid``. The file is opened once;
+        ownership is checked with ``fstat`` on that descriptor, and the token
+        is read from the same descriptor -- never a second, independent
+        lookup of ``config_file`` by name -- so the ownership check and the
+        read always refer to the same file.
         """
         try:
-            st = os.stat(config_file)
+            fd = os.open(config_file, os.O_RDONLY)
         except FileNotFoundError:
             # A missing config file is the normal state for any uid without a
             # Databricks setup (e.g. root or a system daemon probing the mount).
             # Logging it at error level floods the journal with one line per
             # request, so keep it at debug.
             logger.debug("Config file %s does not exist, ignoring", config_file)
-            return False
+            return False, None
         except OSError as exc:
-            # Other stat failures (permission denied, etc.) are unexpected and
+            # Other open failures (permission denied, etc.) are unexpected and
             # worth surfacing, but are not fatal to the mount.
+            logger.warning("Cannot open config file %s: %s", config_file, exc)
+            return False, None
+
+        try:
+            st = os.fstat(fd)
+        except OSError as exc:
             logger.warning("Cannot stat config file %s: %s", config_file, exc)
-            return False
+            os.close(fd)
+            return False, None
         if not stat.S_ISREG(st.st_mode):
             logger.error("Config file %s is not a regular file, ignoring", config_file)
-            return False
+            os.close(fd)
+            return False, None
         if st.st_uid != uid or not (st.st_mode & stat.S_IRUSR):
             logger.error(
                 "Config file %s is not owned and readable by uid %s, ignoring",
                 config_file, uid,
             )
-            return False
-        return True
+            os.close(fd)
+            return False, None
 
-    def _get_profile_and_config_file_name(self, env: dict[str, str]|None, uid: int) -> tuple[str, str] | tuple[None, None]:
-        """ Gets the databricks profile and config file name.
-        1. Check if env defines DATABRICKS_CONFIG_PROFILE and DATABRICKS_CONFIG_FILE
-        2. If not, default to DATABRICKS_CONFIG_PROFILE=DEFAULT and DATABRICKS_CONFIG_FILE=~/.databrickscfg (in the user's home directory)
-
-        Whichever path is selected must be owned by and readable for ``uid``
-        (see ``_is_safe_config_file``); an env-supplied path that fails the
-        check falls through to the default location.
-        """
-        if env is not None:
-            profile = env.get("DATABRICKS_CONFIG_PROFILE", "DEFAULT")
-            config_file = env.get("DATABRICKS_CONFIG_FILE")
-        else:
-            profile = "DEFAULT"
-            config_file = None
-
-        # An env-supplied config file that fails validation is ignored, falling
-        # through to the default location below.
-        if config_file is not None and not self._is_safe_config_file(config_file, uid):
-            config_file = None
-
-        if config_file is None:
-            # Default to ~/.databrickscfg in the user's home directory. This path
-            # is validated identically: under root it may live in another user's
-            # home and must still be owned by the requesting uid.
-            home_dir = self._home_for_uid(uid)
-            default_config = os.path.join(home_dir, ".databrickscfg")
-            if self._is_safe_config_file(default_config, uid):
-                config_file = default_config
-
-        if config_file is None:
-            # Expected for any uid without a Databricks config; debug-level so a
-            # repeatedly-probing uid (e.g. root) does not flood the journal.
-            logger.debug("No databricks config file found for uid %s. Checked environment variable DATABRICKS_CONFIG_FILE and default location", uid)
-            return None, None
-        return profile, config_file
-
-    def _get_token_from_config(self, profile: str, config_file: str) -> Optional[str]:
-        """ Reads the databricks token from the config file for the given profile. """
         config = configparser.ConfigParser()
         try:
-            config.read(config_file)
+            with os.fdopen(fd, "r") as f:
+                config.read_file(f)
         except Exception as exc:
             logger.error("Failed to read databricks config file at %s for profile %s. Exception: %s\n%s", config_file, profile, exc, traceback.format_exc())
-            return None
+            return True, None
+
         if profile not in config:
             logger.error("Profile '%s' not found in %s", profile, config_file)
-            return None
+            return True, None
         try:
-            token = config[profile]["token"]
-            return token
+            return True, config[profile]["token"]
         except KeyError:
             logger.error(f"'token' not found in profile '{profile}' of {config_file}")
-            return None
+            return True, None
 
     async def get_access_token(self, ctx: pyfuse3.RequestContext) -> str|None:
-        """ Gets the databricks token for the given request context by checking the environment variables of the requesting process and then the config file. """
+        """ Gets the databricks token for the given request context by checking the environment variables of the requesting process and then the config file.
+        1. Check if the process defines DATABRICKS_TOKEN in its environment.
+        2. Otherwise, look up a config file: DATABRICKS_CONFIG_PROFILE / DATABRICKS_CONFIG_FILE
+           from the process environment if set, falling back to profile DEFAULT and
+           ~/.databrickscfg in the uid's home directory. An env-supplied config file that
+           fails validation (see ``_read_token_from_path``) falls through to the default.
+        """
         env = self._get_env_for_pid(ctx.pid)
         # The process defines an access token:
         if env is not None and "DATABRICKS_TOKEN" in env:
             return env["DATABRICKS_TOKEN"]
-        # The process does not define a token, but defines a profile or config file:
-        profile, config_file = self._get_profile_and_config_file_name(env, ctx.uid)
-        if profile is None or config_file is None:
-            # Routine when a uid has neither a token in its environment nor a
-            # config file; the caller turns this into EACCES. Debug-level to
-            # avoid one error line per request from a probing uid.
-            logger.debug("No databricks token found in environment variables for pid %s, and no profile or config file found for uid %s", ctx.pid, ctx.uid)
-            return None
-        return self._get_token_from_config(profile, config_file)
+
+        if env is not None:
+            profile = env.get("DATABRICKS_CONFIG_PROFILE", "DEFAULT")
+            config_file = env.get("DATABRICKS_CONFIG_FILE")
+        else:
+            profile = "DEFAULT"
+            config_file = None
+
+        if config_file is not None:
+            found, token = self._read_token_from_path(config_file, ctx.uid, profile)
+            if found:
+                return token
+
+        home_dir = self._home_for_uid(ctx.uid)
+        default_config = os.path.join(home_dir, ".databrickscfg")
+        found, token = self._read_token_from_path(default_config, ctx.uid, profile)
+        if found:
+            return token
+
+        # Expected for any uid without a Databricks config; debug-level so a
+        # repeatedly-probing uid (e.g. root) does not flood the journal.
+        logger.debug("No databricks token found in environment variables for pid %s, and no profile or config file found for uid %s", ctx.pid, ctx.uid)
+        return None
 
 class AuthProvider:
     # Key for the shared token in single-principal mode. -1 is never a valid uid,

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/fs/data_manager.py

@@ -159,14 +159,21 @@ class DataManager:
     async def _process_request(self, chunk_request: _ChunkRequest, priority: str):
         try:
             cache_key = (chunk_request.fs_path, chunk_request.chunk_id, chunk_request.mtime, chunk_request.gen)
-            # We have a chunk_request to download
-            # check if we already downloaded it:
-            chunk = await self.persistence.retrieve_chunk(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime, gen=chunk_request.gen)
-            if chunk is None:
-                # download it.
-                chunk = await self._fetch_chunk(chunk_request)
             if priority == "high":
+                # A real read needs the bytes now, so fetch them (from disk,
+                # falling back to network) and cache in RAM.
+                chunk = await self.persistence.retrieve_chunk(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime, gen=chunk_request.gen)
+                if chunk is None:
+                    chunk = await self._fetch_chunk(chunk_request)
                 await self._ram_cache.put(cache_key, chunk)
+            else:
+                # A prefetch only needs the chunk to end up on disk. Reading
+                # it back (as retrieve_chunk does) would cost as much disk
+                # I/O as the read it's trying to save, for a chunk that may
+                # not even be read; a plain existence check is enough to
+                # decide whether a download is needed.
+                if not await self.persistence.chunk_exists(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime, gen=chunk_request.gen):
+                    await self._fetch_chunk(chunk_request)
         finally:
             await self._inflight_coalescer.notify_done(cache_key)
 
@@ -317,6 +324,3 @@ class DataManager:
             )
             result.extend(chunk_data[chunk_start:chunk_end])
         return bytes(result)
-
-    async def write(self, fs_path: str, offset: int, buffer: bytes, ctx: pyfuse3.RequestContext) -> int:
-        raise pyfuse3.FUSEError(errno.EACCES)

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/fs/inode_manager.py

@@ -134,6 +134,19 @@ class InodeManager:
         if entry.ref_count <= 0:
             self._delete_inode_internal(inode)
 
+    def detach_path(self, fs_path: str) -> None:
+        """Detach ``fs_path`` -- and, if it is a directory, everything
+        beneath it -- from the tree after it has been deleted remotely.
+
+        A no-op if ``fs_path`` has no known inode. This only touches the
+        in-process tree; it does not tell the kernel to drop its own cached
+        attributes/dentries for the path (see
+        ``MetadataManager._invalidate_kernel_cache`` for that).
+        """
+        inode = self.get_inode_by_path(fs_path)
+        if inode is not None:
+            self._prune_subtree(inode)
+
     def add_entry(
         self, parent_inode: int, name: str, attr: InodeEntryAttr
     ):

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/fs/metadata_manager.py

@@ -504,7 +504,7 @@ class MetadataManager:
             if parent_path in self._dir_cache:
                 del self._dir_cache[parent_path]
         except Exception:
-            pass
+            logger.warning("Cache invalidation failed for %s", fs_path, exc_info=True)
 
         # Evicting our own caches is not enough: the kernel keeps its own
         # attribute and dentry caches (for up to attr_timeout / entry_timeout),

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/fs/operations.py

@@ -704,9 +704,7 @@ class UnityCatalogFS(pyfuse3.Operations):
         except Exception as e:
             self._raise_fuse_error(e, fs_path=child_fs_path, op="unlink")
 
-        child_inode = self.inodes.get_inode_by_path(child_fs_path)
-        if child_inode is not None:
-            self.inodes._prune_subtree(child_inode)
+        self.inodes.detach_path(child_fs_path)
         self.metadata_manager.invalidate(child_fs_path, is_dir=False)
         # Drop any cached content so a later same-named file is not served stale.
         await self.data_manager.invalidate_path(child_fs_path)
@@ -734,9 +732,7 @@ class UnityCatalogFS(pyfuse3.Operations):
         except Exception as e:
             self._raise_fuse_error(e, fs_path=child_fs_path, op="rmdir")
 
-        child_inode = self.inodes.get_inode_by_path(child_fs_path)
-        if child_inode is not None:
-            self.inodes._prune_subtree(child_inode)
+        self.inodes.detach_path(child_fs_path)
         self.metadata_manager.invalidate(child_fs_path, is_dir=True)
 
     async def rename(self, parent_inode_old, name_old, parent_inode_new, name_new, flags, ctx):

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/main.py

@@ -283,6 +283,7 @@ async def async_main():
         logger.error("Top-level group message: %s", eg)
         for i, exc in enumerate(eg.exceptions, 1):
             logger.error("[%d] type=%s -> %s\n%s", i, type(exc).__name__, exc, traceback.format_exc())
+        raise
     finally:
         data_manager.close()
         await uc_client.close()

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/fuse4dbricks/storage/persistence.py

@@ -136,6 +136,21 @@ class DiskPersistence:
         with open(path, "rb") as f:
             return f.read()
 
+    async def chunk_exists(
+        self, fs_path: str, chunk_index: int, mtime: float, gen: int = 0
+    ) -> bool:
+        """Whether a chunk is already cached on disk, without reading its
+        content. For a prefetch, the caller only needs to know whether a
+        download is required; reading the full chunk back (as
+        ``retrieve_chunk`` does) would cost as much I/O as the read it is
+        trying to save.
+        """
+        cache_path = self._get_chunk_path(fs_path, chunk_index, mtime, gen)
+        async with self.lock:
+            if cache_path in self.access_map:
+                self.access_map[cache_path] = time.time()
+        return await trio.to_thread.run_sync(os.path.exists, cache_path)
+
     async def store_chunk_from_stream(
         self,
         fs_path: str,

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "fuse4dbricks"
-version = "0.7.4"
+version = "0.7.5"
 description = "FUSE driver for Databricks Unity Catalog Volumes."
 readme = "README.md"
 requires-python = ">=3.11"

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/tests/test_auth_provider.py

@@ -4,8 +4,7 @@ Tests for fuse4dbricks.auth.provider.
 Covers the multi-user trust boundary:
   - _subuid_owner / _home_for_uid   (uid -> home, including subuid ranges)
   - _get_env_for_pid                (binary /proc/<pid>/environ parsing)
-  - _get_profile_and_config_file_name  (ownership/readability checks)
-  - _get_token_from_config          (config parsing)
+  - _read_token_from_path           (atomic open+fstat ownership check, then parse)
   - DatabricksUnifiedAuthProvider.get_access_token  (end-to-end resolution)
   - AuthProvider                    (local-token precedence, invalidation, EACCES)
 """
@@ -115,121 +114,77 @@ def test_get_env_for_pid_parses_and_skips_non_utf8(unified, monkeypatch):
 
 
 # ---------------------------------------------------------------------------
-# _get_profile_and_config_file_name  (ownership / readability)
+# _read_token_from_path  (atomic open+fstat ownership check, then parse)
 # ---------------------------------------------------------------------------
 
 
-def test_profile_config_from_env_owned_by_uid(unified, tmp_path):
+def test_read_token_from_path_owned_by_uid(unified, tmp_path):
     cfg = tmp_path / "cfg"
-    cfg.write_text("[DEFAULT]\ntoken=x\n")
-    uid = os.getuid()
-    env = {"DATABRICKS_CONFIG_PROFILE": "PROD", "DATABRICKS_CONFIG_FILE": str(cfg)}
-    profile, config_file = unified._get_profile_and_config_file_name(env, uid)
-    assert profile == "PROD"
-    assert config_file == str(cfg)
-
-
-def test_profile_config_env_file_not_owned_is_ignored(unified, tmp_path, monkeypatch):
+    cfg.write_text("[PROD]\ntoken = dapi-prod\n")
+    found, token = unified._read_token_from_path(str(cfg), os.getuid(), "PROD")
+    assert (found, token) == (True, "dapi-prod")
+
+
+def test_read_token_from_path_not_owned_is_rejected(unified, tmp_path):
+    # Trust-boundary regression: under root + allow_other, a config file (or
+    # a symlink swapped in to point at another user's file) not owned by the
+    # requesting uid must be rejected — root must not cache the victim's
+    # token under the attacker's uid. Ownership is checked with fstat() on
+    # the same fd that is then read, so there is no separate check-then-open
+    # step for a swapped path to land in between.
     cfg = tmp_path / "cfg"
-    cfg.write_text("[DEFAULT]\ntoken=x\n")
-    # A uid that does not own the temp file -> the env config file is rejected
-    # and we fall back to the home default (which we point at an empty dir).
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "emptyhome"))
-    env = {"DATABRICKS_CONFIG_FILE": str(cfg)}
-    profile, config_file = unified._get_profile_and_config_file_name(env, os.getuid() + 424242)
-    assert (profile, config_file) == (None, None)
+    cfg.write_text("[DEFAULT]\ntoken = victim-secret\n")
+    found, token = unified._read_token_from_path(str(cfg), os.getuid() + 424242, "DEFAULT")
+    assert (found, token) == (False, None)
 
 
-def test_profile_config_env_file_not_regular_is_ignored(unified, tmp_path, monkeypatch):
+def test_read_token_from_path_not_regular_is_rejected(unified, tmp_path):
     d = tmp_path / "adir"
     d.mkdir()
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "emptyhome"))
-    env = {"DATABRICKS_CONFIG_FILE": str(d)}
-    profile, config_file = unified._get_profile_and_config_file_name(env, os.getuid())
-    assert (profile, config_file) == (None, None)
+    found, token = unified._read_token_from_path(str(d), os.getuid(), "DEFAULT")
+    assert (found, token) == (False, None)
 
 
-def test_profile_config_default_home_file(unified, tmp_path, monkeypatch):
-    home = tmp_path / "home"
-    home.mkdir()
-    (home / ".databrickscfg").write_text("[DEFAULT]\ntoken=x\n")
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(home))
-    # The default file must also pass the ownership check, so the resolved uid
-    # has to match the file owner (the test user that just created it).
-    profile, config_file = unified._get_profile_and_config_file_name(env=None, uid=os.getuid())
-    assert profile == "DEFAULT"
-    assert config_file == str(home / ".databrickscfg")
-
-
-def test_profile_config_default_home_file_not_owned_is_rejected(unified, tmp_path, monkeypatch):
-    # Trust-boundary regression: under root + allow_other, the default
-    # ~/.databrickscfg is resolved from another user's home and could be a
-    # symlink the user planted to a victim's config. os.stat follows the link,
-    # so the file appears owned by the victim (not the requesting uid) and must
-    # be rejected — root must not cache the victim's token under the attacker.
-    home = tmp_path / "home"
-    home.mkdir()
-    (home / ".databrickscfg").write_text("[DEFAULT]\ntoken=victim-secret\n")
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(home))
-    # A uid that does not own the file -> rejected, nothing resolved.
-    profile, config_file = unified._get_profile_and_config_file_name(
-        env=None, uid=os.getuid() + 424242
-    )
-    assert (profile, config_file) == (None, None)
-
-
-def test_profile_config_nothing_found(unified, tmp_path, monkeypatch):
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "empty"))
-    profile, config_file = unified._get_profile_and_config_file_name(env=None, uid=os.getuid())
-    assert (profile, config_file) == (None, None)
-
-
-def test_missing_default_config_is_not_logged_as_error(unified, tmp_path, monkeypatch, caplog):
-    """A missing ~/.databrickscfg is the normal state for a uid without a
+def test_read_token_from_path_missing_file_is_not_logged_as_error(unified, tmp_path, caplog):
+    """A missing config file is the normal state for a uid without a
     Databricks setup (e.g. root probing the mount). It must not be logged at
     error level, or a repeatedly-probing uid floods the journal with one line
     per request."""
-    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "empty"))
     with caplog.at_level("DEBUG", logger="fuse4dbricks.auth.provider"):
-        profile, config_file = unified._get_profile_and_config_file_name(env=None, uid=os.getuid())
-    assert (profile, config_file) == (None, None)
-    # The benign "does not exist" path logs at debug, never at error/warning.
+        found, token = unified._read_token_from_path(str(tmp_path / "nope"), os.getuid(), "DEFAULT")
+    assert (found, token) == (False, None)
     assert not [r for r in caplog.records if r.levelname in ("ERROR", "WARNING")]
     assert any(r.levelname == "DEBUG" for r in caplog.records)
 
 
-def test_unreadable_config_is_logged_as_warning(unified, tmp_path, monkeypatch, caplog):
-    """A stat failure that is NOT 'file missing' (e.g. permission denied) is
-    unexpected and stays visible at warning level."""
-    def _boom(path):
+def test_read_token_from_path_open_failure_is_logged_as_warning(unified, tmp_path, monkeypatch, caplog):
+    """An open() failure that is NOT 'file missing' (e.g. permission denied)
+    is unexpected and stays visible at warning level."""
+    def _boom(path, flags):
         raise PermissionError("denied")
-    monkeypatch.setattr(provider_mod.os, "stat", _boom)
+    monkeypatch.setattr(provider_mod.os, "open", _boom)
     with caplog.at_level("DEBUG", logger="fuse4dbricks.auth.provider"):
-        assert unified._is_safe_config_file(str(tmp_path / "cfg"), os.getuid()) is False
+        found, token = unified._read_token_from_path(str(tmp_path / "cfg"), os.getuid(), "DEFAULT")
+    assert (found, token) == (False, None)
     assert any(r.levelname == "WARNING" for r in caplog.records)
 
 
-# ---------------------------------------------------------------------------
-# _get_token_from_config
-# ---------------------------------------------------------------------------
-
-
-def test_get_token_from_config_reads_token(unified, tmp_path):
+def test_read_token_from_path_reads_token(unified, tmp_path):
     cfg = tmp_path / "cfg"
     cfg.write_text("[DEFAULT]\ntoken = dapi-default\n\n[PROD]\ntoken = dapi-prod\n")
-    assert unified._get_token_from_config("PROD", str(cfg)) == "dapi-prod"
+    assert unified._read_token_from_path(str(cfg), os.getuid(), "PROD") == (True, "dapi-prod")
 
 
-def test_get_token_from_config_missing_profile(unified, tmp_path):
+def test_read_token_from_path_missing_profile(unified, tmp_path):
     cfg = tmp_path / "cfg"
     cfg.write_text("[DEFAULT]\ntoken = x\n")
-    assert unified._get_token_from_config("NOPE", str(cfg)) is None
+    assert unified._read_token_from_path(str(cfg), os.getuid(), "NOPE") == (True, None)
 
 
-def test_get_token_from_config_missing_token_key(unified, tmp_path):
+def test_read_token_from_path_missing_token_key(unified, tmp_path):
     cfg = tmp_path / "cfg"
     cfg.write_text("[DEFAULT]\nhost = https://x\n")
-    assert unified._get_token_from_config("DEFAULT", str(cfg)) is None
+    assert unified._read_token_from_path(str(cfg), os.getuid(), "DEFAULT") == (True, None)
 
 
 # ---------------------------------------------------------------------------
@@ -244,21 +199,42 @@ async def test_unified_get_token_prefers_env_token(unified, monkeypatch):
 
 
 @pytest.mark.trio
-async def test_unified_get_token_falls_back_to_config(unified, tmp_path, monkeypatch):
+async def test_unified_get_token_uses_env_config_file(unified, tmp_path, monkeypatch):
     cfg = tmp_path / "cfg"
     cfg.write_text("[DEFAULT]\ntoken = dapi-cfg\n")
-    monkeypatch.setattr(unified, "_get_env_for_pid", lambda pid: {})
+    monkeypatch.setattr(unified, "_get_env_for_pid", lambda pid: {"DATABRICKS_CONFIG_FILE": str(cfg)})
+    assert await unified.get_access_token(_ctx(uid=os.getuid())) == "dapi-cfg"
+
+
+@pytest.mark.trio
+async def test_unified_get_token_missing_env_file_falls_back_to_default(unified, tmp_path, monkeypatch):
+    home = tmp_path / "home"
+    home.mkdir()
+    (home / ".databrickscfg").write_text("[DEFAULT]\ntoken = dapi-home\n")
     monkeypatch.setattr(
-        unified, "_get_profile_and_config_file_name", lambda env, uid: ("DEFAULT", str(cfg))
+        unified, "_get_env_for_pid",
+        lambda pid: {"DATABRICKS_CONFIG_FILE": str(tmp_path / "does-not-exist")},
     )
-    assert await unified.get_access_token(_ctx()) == "dapi-cfg"
+    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(home))
+    assert await unified.get_access_token(_ctx(uid=os.getuid())) == "dapi-home"
+
+
+@pytest.mark.trio
+async def test_unified_get_token_env_file_not_owned_is_ignored(unified, tmp_path, monkeypatch):
+    cfg = tmp_path / "cfg"
+    cfg.write_text("[DEFAULT]\ntoken = x\n")
+    monkeypatch.setattr(unified, "_get_env_for_pid", lambda pid: {"DATABRICKS_CONFIG_FILE": str(cfg)})
+    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "emptyhome"))
+    # A uid that does not own cfg -> the env config file is rejected; the
+    # default home has no config file either -> nothing resolved.
+    assert await unified.get_access_token(_ctx(uid=os.getuid() + 424242)) is None
 
 
 @pytest.mark.trio
-async def test_unified_get_token_none_when_unresolvable(unified, monkeypatch):
+async def test_unified_get_token_none_when_unresolvable(unified, tmp_path, monkeypatch):
     monkeypatch.setattr(unified, "_get_env_for_pid", lambda pid: None)
-    monkeypatch.setattr(unified, "_get_profile_and_config_file_name", lambda env, uid: (None, None))
-    assert await unified.get_access_token(_ctx()) is None
+    monkeypatch.setattr(unified, "_home_for_uid", lambda uid: str(tmp_path / "empty"))
+    assert await unified.get_access_token(_ctx(uid=os.getuid())) is None
 
 
 # ---------------------------------------------------------------------------

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/tests/test_data_manager.py

@@ -14,7 +14,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 import pyfuse3
 
-from fuse4dbricks.fs.data_manager import DataManager
+from fuse4dbricks.fs.data_manager import DataManager, _ChunkRequest
+from fuse4dbricks.fs.ram_cache import RamCache
 
 
 # ---------------------------------------------------------------------------
@@ -285,6 +286,85 @@ async def test_read_prefetch_stops_at_eof(manager, ctx):
     assert 3 not in prefetched_chunk_ids  # chunk 3 doesn't exist
 
 
+# ---------------------------------------------------------------------------
+# Tests: _process_request (on-demand reads vs. prefetch downloads)
+#
+# On-demand ("high" priority) reads need the actual bytes now, so they read
+# the chunk (from disk, falling back to network) and cache it in RAM.
+# Prefetch ("regular" priority) only needs the chunk to end up on disk --
+# reading it back would cost as much disk I/O as the read it's trying to
+# save, for a chunk that may never be read, so it uses a cheap existence
+# check and never populates the RAM cache.
+# ---------------------------------------------------------------------------
+
+
+def _chunk_request(manager, chunk_id=0, mtime=100.0, gen=0):
+    return _ChunkRequest(
+        fs_path="/c/s/v/f", chunk_id=chunk_id, mtime=mtime, gen=gen,
+        chunk_size=manager.chunk_size, ctx=SimpleNamespace(uid=1000, pid=1, gid=1000),
+    )
+
+
+@pytest.mark.trio
+async def test_process_request_high_priority_disk_hit_caches_in_ram(manager):
+    # The `manager` fixture's ram_cache_mb=1 rounds down to 0-entry capacity
+    # (1 MB < one 8 MB chunk), which would no-op every put/get below.
+    manager._ram_cache = RamCache(max_entries=8)
+    manager.persistence.retrieve_chunk = AsyncMock(return_value=b"cached-bytes")
+    request = _chunk_request(manager)
+
+    await manager._process_request(request, priority="high")
+
+    manager.persistence.store_chunk_from_stream.assert_not_awaited()
+    cached = await manager._ram_cache.get(("/c/s/v/f", 0, 100.0, 0))
+    assert cached == b"cached-bytes"
+
+
+@pytest.mark.trio
+async def test_process_request_high_priority_miss_downloads_and_caches_in_ram(manager):
+    manager._ram_cache = RamCache(max_entries=8)
+    manager.persistence.retrieve_chunk = AsyncMock(return_value=None)
+    manager.persistence.store_chunk_from_stream = AsyncMock(return_value=b"downloaded")
+    request = _chunk_request(manager)
+
+    await manager._process_request(request, priority="high")
+
+    manager.persistence.store_chunk_from_stream.assert_awaited_once()
+    cached = await manager._ram_cache.get(("/c/s/v/f", 0, 100.0, 0))
+    assert cached == b"downloaded"
+
+
+@pytest.mark.trio
+async def test_process_request_prefetch_skips_download_when_already_on_disk(manager):
+    manager.persistence.chunk_exists = AsyncMock(return_value=True)
+    request = _chunk_request(manager, chunk_id=5)
+
+    await manager._process_request(request, priority="regular")
+
+    manager.persistence.chunk_exists.assert_awaited_once_with(
+        fs_path="/c/s/v/f", chunk_index=5, mtime=100.0, gen=0
+    )
+    # No download, and -- crucially -- no read of the chunk's content either
+    # (that would cost the same disk I/O the existence check is avoiding).
+    manager.persistence.retrieve_chunk.assert_not_awaited()
+    manager.persistence.store_chunk_from_stream.assert_not_awaited()
+    assert await manager._ram_cache.get(("/c/s/v/f", 5, 100.0, 0)) is None
+
+
+@pytest.mark.trio
+async def test_process_request_prefetch_downloads_when_missing_but_not_cached_in_ram(manager):
+    manager.persistence.chunk_exists = AsyncMock(return_value=False)
+    manager.persistence.store_chunk_from_stream = AsyncMock(return_value=b"downloaded")
+    request = _chunk_request(manager, chunk_id=5)
+
+    await manager._process_request(request, priority="regular")
+
+    manager.persistence.retrieve_chunk.assert_not_awaited()
+    manager.persistence.store_chunk_from_stream.assert_awaited_once()
+    # Prefetched content is deliberately not promoted into the RAM cache.
+    assert await manager._ram_cache.get(("/c/s/v/f", 5, 100.0, 0)) is None
+
+
 # ---------------------------------------------------------------------------
 # Tests: cache generation (invalidate_path)
 # ---------------------------------------------------------------------------

{fuse4dbricks-0.7.4 → fuse4dbricks-0.7.5}/tests/test_persistence.py

@@ -110,6 +110,33 @@ async def test_retrieve_missing_chunk(persistence):
     assert result is None
 
 
+@pytest.mark.trio
+async def test_chunk_exists_true_after_store(persistence):
+    """chunk_exists reports a stored chunk as present, without needing to
+    read its content."""
+    await persistence.store_chunk_from_stream("f", 0, 5.0, async_byte_generator(b"data"))
+    assert await persistence.chunk_exists("f", 0, 5.0) is True
+
+
+@pytest.mark.trio
+async def test_chunk_exists_false_for_missing_chunk(persistence):
+    assert await persistence.chunk_exists("ghost_file", 99, 0.0) is False
+
+
+@pytest.mark.trio
+async def test_chunk_exists_does_not_read_file_content(persistence, monkeypatch):
+    """chunk_exists must not pay the I/O cost of reading the chunk back --
+    that's the whole point of using it instead of retrieve_chunk for a
+    prefetch's "is a download needed" check."""
+    await persistence.store_chunk_from_stream("f", 0, 5.0, async_byte_generator(b"data"))
+
+    def _boom(path):
+        raise AssertionError("chunk_exists must not read the file's content")
+
+    monkeypatch.setattr(persistence, "_read_file", _boom)
+    assert await persistence.chunk_exists("f", 0, 5.0) is True
+
+
 @pytest.mark.trio
 async def test_get_chunk_path_is_pure_no_dir_created(persistence):
     """_get_chunk_path must not touch the filesystem: it runs on every read,