fuse4dbricks 0.7.3__tar.gz → 0.7.4__tar.gz

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/CHANGELOG.md

@@ -1,3 +1,34 @@
+# 0.7.4 (2026-06-20)
+
+- Make writes durable and immediately visible on `close()`. The upload ran only
+  in `release()`, which the kernel does not wait for and whose errors it
+  discards, so `close()` returned before the data was uploaded — a file written
+  through the mount was invisible (and write errors were silent) for the
+  duration of the asynchronous upload. The upload now runs in `flush()`, which
+  handles the `close()` syscall: the kernel blocks on it, the upload completes
+  before `close()` returns, and a failed upload surfaces as a `close()` error.
+  `release()` keeps the upload as a fallback for the rare close-without-flush.
+- Fix a stale read after overwriting a file within the same second. Cached
+  chunks are keyed by the server `mtime`, but the Files API `Last-Modified`
+  header has only 1-second resolution (verified against the live API: no
+  sub-second timestamp and no `ETag`), so two writes in the same second shared a
+  key and a read could return the previous content. The chunk cache key now
+  carries a per-path generation counter (RAM key and on-disk chunk filename)
+  that `DataManager.invalidate_path()` bumps O(1) on every local write (wired
+  into write/flush, rename, truncate and unlink); old chunks become unreachable
+  and age out via the existing LRUs. Exposed by the durable-`close()` fix above,
+  which made same-second overwrites reachable.
+- Fix cache invalidation not reaching the kernel. Mutating ops route through
+  `invalidate()`, which evicted the in-process attribute/dir/negative caches but
+  never told the FUSE kernel module, so the kernel kept serving stale `stat`
+  results from its own attribute and dentry caches for up to its
+  `attr_timeout` / `entry_timeout` (the pyfuse3 default, 300s) — a deleted file
+  kept stat-ing OK and a recreated one stayed hidden. `invalidate()` now also
+  resolves the path through the inode manager and calls
+  `pyfuse3.invalidate_entry_async()` (drop the parent dentry; the async variant
+  avoids deadlocking against the unlink/rename/create request in flight) and
+  `pyfuse3.invalidate_inode()` (drop the inode's cached attributes and data).
+
 # 0.7.3 (2026-06-19)
 
 - Fix `mv` of a freshly written file failing with an I/O error. `rename()`

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fuse4dbricks
-Version: 0.7.3
+Version: 0.7.4
 Summary: FUSE driver for Databricks Unity Catalog Volumes.
 Project-URL: Homepage, https://github.com/zeehio/fuse4dbricks
 Author-email: Sergio Oller <sergioller@gmail.com>

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/fs/data_manager.py

@@ -24,16 +24,24 @@ class _ChunkRequest:
     "piece to download"
     mtime: float
     "last modified, to ensure chunks are not altered between reads"
+    gen: int
+    "cache generation: bumped on a local write so a same-second overwrite (mtime has 1s resolution) is not served stale"
     chunk_size: int
     "size of this chunk. It may be smaller than the typical chunk_size on the last chunk of o a file"
     ctx: pyfuse3.RequestContext
     "request context, used for auth when downloading"
 
+
+# Chunk cache key: (fs_path, chunk_id, mtime, gen). mtime guards against the
+# remote file changing under us; gen is a local epoch bumped on every write so a
+# stale chunk is unreachable even when the server mtime (1s resolution) collides.
+_CacheKey = Tuple[str, int, float, int]
+
 class DownloadScheduler:
     def __init__(self, process_request, num_workers=10):
         self._requests_lock = trio.Lock()
-        self._requests_priority: OrderedDict[Tuple[str, int, float], _ChunkRequest] = OrderedDict()
-        self._requests_regular: OrderedDict[Tuple[str, int, float], _ChunkRequest] = OrderedDict()
+        self._requests_priority: OrderedDict[_CacheKey, _ChunkRequest] = OrderedDict()
+        self._requests_regular: OrderedDict[_CacheKey, _ChunkRequest] = OrderedDict()
         self._requests_num_workers = num_workers
         self._process_request = process_request
 
@@ -43,7 +51,7 @@ class DownloadScheduler:
         for i in range(self._requests_num_workers):
             nursery.start_soon(self._downloader)
 
-    async def _dequeue_one(self) -> tuple[Tuple[str, int, float] | None, _ChunkRequest | None, str]:
+    async def _dequeue_one(self) -> tuple[_CacheKey | None, _ChunkRequest | None, str]:
         async with self._requests_lock:
             try:
                 cache_key, chunk_request = self._requests_priority.popitem(last=False)
@@ -92,7 +100,7 @@ class DownloadScheduler:
             pass
 
     async def enqueue_request(self, chunk_request: _ChunkRequest, high_priority: bool):
-        cache_key = (chunk_request.fs_path, chunk_request.chunk_id, chunk_request.mtime)
+        cache_key = (chunk_request.fs_path, chunk_request.chunk_id, chunk_request.mtime, chunk_request.gen)
         async with self._requests_lock:
             if high_priority:
                 # If request for that chunk was present as a regular request, promote it:
@@ -118,24 +126,42 @@ class DataManager:
         self.uc_client = uc_client
         self.persistence = persistence
         self.chunk_size = 8 * 1024 * 1024
-        self._inflight_coalescer: InflightCoalescer[Tuple[str, int, float]] = InflightCoalescer()
-        "Inflight coalescer for downloads identified by (file_id, chunk_id, mtime)"
+        self._inflight_coalescer: InflightCoalescer[_CacheKey] = InflightCoalescer()
+        "Inflight coalescer for downloads identified by (fs_path, chunk_id, mtime, gen)"
 
         self._download_scheduler = DownloadScheduler(process_request=self._process_request, num_workers=num_workers)
 
+        # Per-path cache generation. Bumped on a local write so previously cached
+        # chunks become unreachable without scanning. Absent path => generation 0.
+        self._write_generation: dict[str, int] = {}
+
         # max number of chunks to cache in memory
         max_ram_chunks = int(ram_cache_mb*1024*1024 / self.chunk_size)
-        self._ram_cache: RamCache[Tuple[str, int, float]] = RamCache(max_entries = max_ram_chunks)
+        self._ram_cache: RamCache[_CacheKey] = RamCache(max_entries = max_ram_chunks)
 
     def close(self) -> None:
         self._download_scheduler.close()
 
+    def _generation(self, fs_path: str) -> int:
+        return self._write_generation.get(fs_path, 0)
+
+    async def invalidate_path(self, fs_path: str) -> None:
+        """Drop the cached content of ``fs_path`` after a local write.
+
+        O(1): bumps the path's cache generation rather than scanning RAM or
+        globbing disk. Subsequent reads compute a new key, so the old chunks are
+        unreachable and age out via the RAM/disk LRUs. Necessary because chunks
+        are keyed by the server ``mtime``, which has only 1-second resolution and
+        so cannot distinguish two writes made within the same second.
+        """
+        self._write_generation[fs_path] = self._generation(fs_path) + 1
+
     async def _process_request(self, chunk_request: _ChunkRequest, priority: str):
         try:
-            cache_key = (chunk_request.fs_path, chunk_request.chunk_id, chunk_request.mtime)
+            cache_key = (chunk_request.fs_path, chunk_request.chunk_id, chunk_request.mtime, chunk_request.gen)
             # We have a chunk_request to download
             # check if we already downloaded it:
-            chunk = await self.persistence.retrieve_chunk(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime)
+            chunk = await self.persistence.retrieve_chunk(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime, gen=chunk_request.gen)
             if chunk is None:
                 # download it.
                 chunk = await self._fetch_chunk(chunk_request)
@@ -164,39 +190,40 @@ class DataManager:
             fs_path=chunk_request.fs_path,
             chunk_index=chunk_request.chunk_id,
             mtime=chunk_request.mtime,
+            gen=chunk_request.gen,
             stream=stream,
         )
         return chunk
 
-    async def _request_fetch_ahead_chunks(self, fs_path: str, chunks_to_prefetch: list[Tuple[int, int]], mtime: float, ctx: pyfuse3.RequestContext):
+    async def _request_fetch_ahead_chunks(self, fs_path: str, chunks_to_prefetch: list[Tuple[int, int]], mtime: float, gen: int, ctx: pyfuse3.RequestContext):
         for (chunk_id, chunk_size) in chunks_to_prefetch:
-            cache_key = (fs_path, chunk_id, mtime)
+            cache_key = (fs_path, chunk_id, mtime, gen)
             (wait_event, leader) = await self._inflight_coalescer.join_or_lead(cache_key)
             if not leader:
                 continue  # already being fetched, we don't care about it
-            chunk_request = _ChunkRequest(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, chunk_size=chunk_size, ctx=ctx)
+            chunk_request = _ChunkRequest(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, gen=gen, chunk_size=chunk_size, ctx=ctx)
             await self._download_scheduler.enqueue_request(chunk_request, high_priority=False)
         return None
 
-    async def _get_chunk_from_cache_or_disk(self, fs_path: str, chunk_id: int, mtime: float) -> bytes | None:
-        cache_key = (fs_path, chunk_id, mtime)
+    async def _get_chunk_from_cache_or_disk(self, fs_path: str, chunk_id: int, mtime: float, gen: int) -> bytes | None:
+        cache_key = (fs_path, chunk_id, mtime, gen)
         # get chunk from ram
         cached = await self._ram_cache.get(cache_key)
         if cached is not None:
             return cached
         # get chunk from disk
-        chunk = await self.persistence.retrieve_chunk(fs_path, chunk_id, mtime)
+        chunk = await self.persistence.retrieve_chunk(fs_path, chunk_id, mtime, gen)
         if chunk is not None:
             await self._ram_cache.put(cache_key, chunk)
         return chunk
 
 
     async def _read_chunk(
-        self, fs_path: str, chunk_id: int, mtime: float, chunk_size: int, 
+        self, fs_path: str, chunk_id: int, mtime: float, gen: int, chunk_size: int,
         ctx: pyfuse3.RequestContext, out_dict: dict[int, bytes|None]
     ):
-        cache_key = (fs_path, chunk_id, mtime)
-        chunk = await self._get_chunk_from_cache_or_disk(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime)
+        cache_key = (fs_path, chunk_id, mtime, gen)
+        chunk = await self._get_chunk_from_cache_or_disk(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, gen=gen)
         if chunk is not None:
             out_dict[chunk_id] = chunk
             return
@@ -206,12 +233,12 @@ class DataManager:
 
         if leader:
             # get chunk from network
-            chunk_request = _ChunkRequest(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, chunk_size=chunk_size, ctx=ctx)
+            chunk_request = _ChunkRequest(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, gen=gen, chunk_size=chunk_size, ctx=ctx)
             await self._download_scheduler.enqueue_request(chunk_request, high_priority=True)
 
         # Now we wait for the download to complete and fetch it from RAM or Disk
         await wait_event.wait()
-        chunk = await self._get_chunk_from_cache_or_disk(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime)
+        chunk = await self._get_chunk_from_cache_or_disk(fs_path=fs_path, chunk_id=chunk_id, mtime=mtime, gen=gen)
         # here if chunk is none, we will return None and let the caller decide what to do (probably return EIO)
         out_dict[chunk_id] = chunk
         return
@@ -224,6 +251,9 @@ class DataManager:
         Reads file contents
         Raises on error raise pyfuse3.FUSEError(errno.EIO)
         """
+        # Resolve the cache generation once for the whole read so every chunk
+        # (and the prefetch below) shares a consistent key.
+        gen = self._generation(fs_path)
         # Determine path and chunks that need to be read
         if offset >= file_size:
             return bytes()
@@ -246,7 +276,7 @@ class DataManager:
                 chunk_size = last_chunk_size
             else:
                 chunk_size = self.chunk_size
-            await self._read_chunk(fs_path, chunk_id, mtime, chunk_size, ctx, chunks)
+            await self._read_chunk(fs_path, chunk_id, mtime, gen, chunk_size, ctx, chunks)
         else:
             # dictionary: chunk_id -> bytes
             async with trio.open_nursery() as nursery:
@@ -256,7 +286,7 @@ class DataManager:
                     else:
                         chunk_size = self.chunk_size
                     nursery.start_soon(
-                        self._read_chunk, fs_path, chunk_id, mtime, chunk_size, ctx, chunks
+                        self._read_chunk, fs_path, chunk_id, mtime, gen, chunk_size, ctx, chunks
                     )
         # fetch ahead next 10 chunks once we are beyond chunk 0 (avoid prefetching too much on `head *` command)
         num_chunks_to_prefetch = 10 if end_chunk > 0 else 1
@@ -273,7 +303,7 @@ class DataManager:
                 chunk_size = self.chunk_size
             # Add to list to fetch:
             chunks_to_prefetch.append((chunk_to_prefetch, chunk_size))
-        await self._request_fetch_ahead_chunks(fs_path, chunks_to_prefetch, mtime, ctx)
+        await self._request_fetch_ahead_chunks(fs_path, chunks_to_prefetch, mtime, gen, ctx)
         # Assemble chunks
         result = bytearray()
         for chunk_id in chunks_to_read:

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/fs/metadata_manager.py

@@ -15,7 +15,7 @@ import pyfuse3
 
 from fuse4dbricks.api.uc_client import (UcNodeType, UnityCatalogClient,
                                         UnityCatalogEntry)
-from fuse4dbricks.fs.inode_manager import InodeEntry, InodeEntryAttr
+from fuse4dbricks.fs.inode_manager import InodeEntry, InodeEntryAttr, InodeManager
 from fuse4dbricks.fs.utils import (fs_to_uc_path,  fs_to_securable,
                                    InflightCoalescer,
                                    uc_to_fs_path)
@@ -54,6 +54,7 @@ class MetadataManager:
         max_entries=20000,
         ttl_catalog=600.0,
         ttl_negative: float=5.0,
+        inode_manager: InodeManager | None = None,
     ):
         """
         :param uc_client: The Unity Catalog API client.
@@ -61,8 +62,15 @@ class MetadataManager:
         :param max_entries: Maximum number of metadata entries to keep in RAM.
         :param ttl_negative: Time-to-live in seconds for negative (not-found)
             cache entries. Kept short because it hides newly-created paths.
+        :param inode_manager: Used by ``invalidate`` to resolve an fs_path to an
+            inode so the kernel's own attribute/dentry cache can be dropped too.
+            Optional: when ``None`` only the in-process caches are evicted (the
+            kernel still expires its entries on its own ``attr_timeout`` /
+            ``entry_timeout``). Always wired in production; left unset in unit
+            tests that exercise the in-process caches in isolation.
         """
         self.uc_client = uc_client
+        self._inode_manager = inode_manager
         self._ttl: float = ttl
         self._ttl_catalog = ttl_catalog
         self._ttl_negative = ttl_negative
@@ -498,6 +506,59 @@ class MetadataManager:
         except Exception:
             pass
 
+        # Evicting our own caches is not enough: the kernel keeps its own
+        # attribute and dentry caches (for up to attr_timeout / entry_timeout),
+        # so without this a deleted file would keep stat-ing OK and a fresh one
+        # would stay hidden until the kernel's timeout expired. Tell the kernel
+        # to forget this path too.
+        self._invalidate_kernel_cache(fs_path)
+
+    def _invalidate_kernel_cache(self, fs_path: str):
+        """Drop the kernel's cached attributes, data and dentry for ``fs_path``.
+
+        Resolves the path through the inode manager and notifies the FUSE
+        kernel module. A no-op when no inode manager is wired (unit tests) or
+        when neither the path nor its parent is currently known to the kernel.
+        """
+        if self._inode_manager is None:
+            return
+        norm = fs_path.rstrip("/") or "/"
+        if norm == "/":
+            return
+        parent_path, _, name = norm.rpartition("/")
+        parent_path = parent_path or "/"
+
+        # Forget the directory entry in the parent so a deleted name stops
+        # resolving and a recreated one is re-looked-up. Must use the *_async
+        # variant: invalidate() runs inside the very unlink/rename/create
+        # request that touched this entry, and the synchronous invalidate_entry
+        # would deadlock against it.
+        parent_inode = self._inode_manager.get_inode_by_path(parent_path)
+        if parent_inode is not None:
+            try:
+                pyfuse3.invalidate_entry_async(
+                    parent_inode, name.encode("utf-8"), ignore_enoent=True
+                )
+            except Exception:
+                logger.debug(
+                    "invalidate_entry_async failed for %s", fs_path, exc_info=True
+                )
+
+        # Forget cached attributes and page data for the inode itself, so a
+        # changed size/mtime or stale contents are re-fetched on next access.
+        inode = self._inode_manager.get_inode_by_path(norm)
+        if inode is not None:
+            try:
+                pyfuse3.invalidate_inode(inode)
+            except OSError:
+                # ENOSYS (kernel lacks support) or a transient failure: the
+                # kernel's own attr_timeout still bounds how long it stays stale.
+                pass
+            except Exception:
+                logger.debug(
+                    "invalidate_inode failed for %s", fs_path, exc_info=True
+                )
+
     # =========================================================================
     # Internal Logic & Helpers
     # =========================================================================

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/fs/operations.py

@@ -473,6 +473,55 @@ class UnityCatalogFS(pyfuse3.Operations):
         }
         return (pyfuse3.FileInfo(fh=fh), self._entry_to_fuse_attr(entry))
 
+    async def _upload_if_dirty(self, state: dict) -> None:
+        """Upload the write buffer to Unity Catalog if it has unsaved writes.
+
+        Shared by ``flush`` and ``release``. Uploads, then marks the handle
+        clean and busts the caches so the new content is immediately visible.
+        Raises ``FUSEError(EIO)`` on upload failure (the caller decides whether
+        that error reaches userspace). A no-op when the buffer is clean, so a
+        second ``flush`` — or the ``release`` fallback after a successful
+        ``flush`` — does not re-upload.
+        """
+        write_buffer: WriteBuffer | None = state.get("write_buffer")
+        if write_buffer is None or not state.get("dirty", False):
+            return
+
+        entry = self.inodes.get_entry(state["inode"])
+        if entry is None:
+            raise pyfuse3.FUSEError(errno.ENOENT)
+
+        uc_path = fs_to_uc_path(entry.fs_path)
+        try:
+            # Push buffered writes to the OS so the upload's separate reader sees
+            # the complete file (writes smaller than ~8 KB would otherwise read
+            # back as zero bytes). Keep the handle open: flush may be delivered
+            # again with more writes before release.
+            write_buffer.flush_to_disk()
+            await self.uc_client.upload_file(uc_path, write_buffer.path, ctx=state["ctx"])
+        except Exception as e:
+            logger.error("Upload failed for %s: %s", entry.fs_path, e)
+            # Remote file is unchanged: do NOT invalidate the cache.
+            raise pyfuse3.FUSEError(errno.EIO) from e
+        # Clean before invalidating so a concurrent reader never sees "dirty".
+        state["dirty"] = False
+        # Bust caches so subsequent reads see the new file: metadata (size/mtime)
+        # and the chunk content cache (whose mtime key cannot tell apart two
+        # writes in the same second).
+        self.metadata_manager.invalidate(entry.fs_path, is_dir=False)
+        await self.data_manager.invalidate_path(entry.fs_path)
+
+    async def flush(self, fh: pyfuse3.FileHandleT) -> None:
+        # flush handles the close() syscall: the kernel waits for it and returns
+        # its error to userspace. Uploading here (rather than only in release(),
+        # which runs asynchronously and whose errors the kernel discards) makes a
+        # write durable and immediately visible by the time close() returns, and
+        # lets a failed upload surface as a close() error instead of being lost.
+        state = self._open_state.get(fh)
+        if state is None:
+            return
+        await self._upload_if_dirty(state)
+
     async def release(self, fh: pyfuse3.FileHandleT) -> None:
         if fh not in self._open_state:
             # This should not happen, but we want to be resilient to it. Just ignore.
@@ -482,7 +531,6 @@ class UnityCatalogFS(pyfuse3.Operations):
         inode = state["inode"]
         ctx = state["ctx"]
         write_buffer: WriteBuffer | None = state.get("write_buffer")
-        dirty: bool = state.get("dirty", False)
 
         entry = self.inodes.get_entry(inode)
         if entry is None:
@@ -492,26 +540,11 @@ class UnityCatalogFS(pyfuse3.Operations):
             raise pyfuse3.FUSEError(errno.ENOENT)
 
         try:
-            if write_buffer is not None and dirty:
-                uc_path = fs_to_uc_path(entry.fs_path)
-                try:
-                    # Flush + close the write handle so the upload reads the
-                    # complete file from disk (buffered writes smaller than
-                    # ~8 KB would otherwise upload as zero bytes).
-                    write_buffer.finalize()
-                    await self.uc_client.upload_file(
-                        uc_path, write_buffer.path, ctx=ctx
-                    )
-                except Exception as e:
-                    logger.error("Upload failed for %s: %s", entry.fs_path, e)
-                    # Remote file is unchanged: do NOT invalidate the cache.
-                    raise pyfuse3.FUSEError(errno.EIO) from e
-                else:
-                    # Bust caches so subsequent reads see the new file.
-                    self.metadata_manager.invalidate(entry.fs_path, is_dir=False)
-            elif write_buffer is not None:
-                # Opened writable but never written — discard buffer silently.
-                pass
+            if write_buffer is not None:
+                # Normally flush() already uploaded and cleared the dirty flag,
+                # making this a no-op; it stays as a fallback for the rare path
+                # where release arrives without a preceding flush.
+                await self._upload_if_dirty(state)
             elif self._dispatch(entry.fs_path) == "auth":
                 try:
                     await self.auth_manager.release(entry.fs_path, ctx=ctx)
@@ -675,6 +708,8 @@ class UnityCatalogFS(pyfuse3.Operations):
         if child_inode is not None:
             self.inodes._prune_subtree(child_inode)
         self.metadata_manager.invalidate(child_fs_path, is_dir=False)
+        # Drop any cached content so a later same-named file is not served stale.
+        await self.data_manager.invalidate_path(child_fs_path)
 
     async def rmdir(self, parent_inode, name, ctx):
         if self._read_only:
@@ -829,6 +864,10 @@ class UnityCatalogFS(pyfuse3.Operations):
         self.inodes.move_inode(src_inode, parent_inode_new, new_name)
         self.metadata_manager.invalidate(old_fs_path, is_dir=False)
         self.metadata_manager.invalidate(new_fs_path, is_dir=False)
+        # The source is gone and the destination has new content; drop both from
+        # the chunk content cache.
+        await self.data_manager.invalidate_path(old_fs_path)
+        await self.data_manager.invalidate_path(new_fs_path)
 
     async def setattr(self, inode, attr, fields, fh, ctx):
         entry = self.inodes.get_entry(inode)
@@ -940,6 +979,7 @@ class UnityCatalogFS(pyfuse3.Operations):
             write_buffer.close()
         entry.attr.st_size = new_size
         self.metadata_manager.invalidate(entry.fs_path, is_dir=False)
+        await self.data_manager.invalidate_path(entry.fs_path)
 
     async def forget(self, inode_list):
         for inode, nlookup in inode_list:

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/fs/write_buffer.py

@@ -48,6 +48,19 @@ class WriteBuffer:
         self._tmp.truncate(size)
         self._size = size
 
+    def flush_to_disk(self) -> None:
+        """Push buffered writes to the OS so a separate reader sees them.
+
+        Like ``finalize()`` this makes the full content visible to the upload's
+        own ``open(self.path)``, but it keeps the write handle open so ``read()``
+        and ``write()`` may continue afterwards. Use this when the upload happens
+        on ``flush`` (the ``close()`` syscall), which the kernel may deliver more
+        than once for a single open file (e.g. a duplicated fd) with further
+        writes in between. Idempotent; a no-op once the handle is closed.
+        """
+        if not self._tmp.closed:
+            self._tmp.flush()
+
     def finalize(self) -> None:
         """Flush and close the write handle, keeping the file on disk.
 

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/main.py

@@ -243,7 +243,7 @@ async def async_main():
 
     inode_manager = InodeManager()
     data_manager = DataManager(uc_client, persistence, ram_cache_mb=args.ram_cache_mb)
-    metadata_manager = MetadataManager(uc_client, ttl=args.metadata_cache_ttl_sec, max_entries=args.metadata_cache_max_entries, ttl_catalog=args.metadata_cache_ttl_catalog_sec, ttl_negative=args.metadata_cache_ttl_negative_sec)
+    metadata_manager = MetadataManager(uc_client, ttl=args.metadata_cache_ttl_sec, max_entries=args.metadata_cache_max_entries, ttl_catalog=args.metadata_cache_ttl_catalog_sec, ttl_negative=args.metadata_cache_ttl_negative_sec, inode_manager=inode_manager)
     # On a 401, the token is invalidated and re-resolved (possibly to a
     # different principal); drop the cached principal so authz re-derives it.
     auth_provider.set_token_invalidation_callback(metadata_manager.forget_principal)

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/fuse4dbricks/storage/persistence.py

@@ -43,7 +43,7 @@ class DiskPersistence:
         nursery.start_soon(self._graceful_init)
         nursery.start_soon(self._background_maintenance)
 
-    def _get_chunk_path(self, fs_path: str, chunk_index: int, mtime: float) -> str:
+    def _get_chunk_path(self, fs_path: str, chunk_index: int, mtime: float, gen: int = 0) -> str:
         """Compute the cache path for a chunk (256-shard distribution).
 
         Pure: does NOT touch the filesystem. ``retrieve_chunk`` runs on every
@@ -59,9 +59,10 @@ class DiskPersistence:
         shard2 = f"{(chunk_index // 1000):07d}"  # Secondary shard to prevent too many files in one dir
         shard_dir = os.path.join(self.cache_dir, shard1, shard2)
         mtime_ms = int(mtime * 1000)
-        # the cache_path
+        # gen is a local cache epoch (see DataManager.invalidate_path); it keeps
+        # a same-second overwrite from reusing a stale chunk file on disk.
         return os.path.join(
-            shard_dir, f"{sha256_hash}_{mtime_ms}_{chunk_index:07d}.bin"
+            shard_dir, f"{sha256_hash}_{mtime_ms}_g{gen}_{chunk_index:07d}.bin"
         )
 
     @staticmethod
@@ -96,13 +97,13 @@ class DiskPersistence:
                     continue
 
     async def retrieve_chunk(
-        self, fs_path: str, chunk_index: int, mtime: float
+        self, fs_path: str, chunk_index: int, mtime: float, gen: int = 0
     ) -> bytes | None:
         """
         Retrieves a chunk from disk.
         Updates LRU access time BEFORE reading to prevent concurrent eviction.
         """
-        cache_path = self._get_chunk_path(fs_path, chunk_index, mtime)
+        cache_path = self._get_chunk_path(fs_path, chunk_index, mtime, gen)
 
         # 1. OPTIMISTIC PROMOTION (Pinning)
         # Mark as accessed so GC doesn't delete it while we read.
@@ -141,9 +142,10 @@ class DiskPersistence:
         chunk_index: int,
         mtime: float,
         stream: AsyncGenerator[bytes, None],
+        gen: int = 0,
     ) -> bytes:
         """Consumes a stream and writes it to disk. Returns the bytes written."""
-        cache_path = self._get_chunk_path(fs_path, chunk_index, mtime)
+        cache_path = self._get_chunk_path(fs_path, chunk_index, mtime, gen)
         temp_path = f"{cache_path}.{os.getpid()}.tmp"
         result = bytearray()
         bytes_written = 0

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "fuse4dbricks"
-version = "0.7.3"
+version = "0.7.4"
 description = "FUSE driver for Databricks Unity Catalog Volumes."
 readme = "README.md"
 requires-python = ">=3.11"

{fuse4dbricks-0.7.3 → fuse4dbricks-0.7.4}/tests/test_data_manager.py

@@ -48,7 +48,7 @@ def manager():
 
 def _make_fake_read_chunk(chunk_size: int, fill_byte: bytes = b"x"):
     """Returns a coroutine that writes `chunk_size` bytes of fill_byte to out_dict[chunk_id]."""
-    async def _fake(fs_path, chunk_id, mtime, cs, ctx, out_dict):
+    async def _fake(fs_path, chunk_id, mtime, gen, cs, ctx, out_dict):
         out_dict[chunk_id] = fill_byte * cs
 
     return _fake
@@ -56,7 +56,7 @@ def _make_fake_read_chunk(chunk_size: int, fill_byte: bytes = b"x"):
 
 def _make_fake_read_chunk_none():
     """Returns a coroutine that sets out_dict[chunk_id] = None (simulates download failure)."""
-    async def _fake(fs_path, chunk_id, mtime, chunk_size, ctx, out_dict):
+    async def _fake(fs_path, chunk_id, mtime, gen, chunk_size, ctx, out_dict):
         out_dict[chunk_id] = None
 
     return _fake
@@ -159,7 +159,7 @@ async def test_read_last_chunk_smaller_than_chunk_size(manager, ctx):
 
     call_log: list[int] = []
 
-    async def fake_read_chunk(fs_path, chunk_id, mtime, cs, ctx, out_dict):
+    async def fake_read_chunk(fs_path, chunk_id, mtime, gen, cs, ctx, out_dict):
         call_log.append((chunk_id, cs))
         out_dict[chunk_id] = b"x" * cs
 
@@ -184,7 +184,7 @@ async def test_read_file_size_exact_multiple_of_chunk_size(manager, ctx):
 
     call_log: list[tuple[int, int]] = []
 
-    async def fake_read_chunk(fs_path, chunk_id, mtime, cs, ctx, out_dict):
+    async def fake_read_chunk(fs_path, chunk_id, mtime, gen, cs, ctx, out_dict):
         call_log.append((chunk_id, cs))
         out_dict[chunk_id] = b"x" * cs
 
@@ -230,7 +230,7 @@ async def test_read_prefetch_not_triggered_at_start_of_first_chunk(manager, ctx)
     file_size = 12 * CHUNK
     prefetch_calls: list = []
 
-    async def fake_prefetch(fs_path, chunks, mtime, ctx):
+    async def fake_prefetch(fs_path, chunks, mtime, gen, ctx):
         prefetch_calls.append(chunks)
 
     with patch.object(manager, "_read_chunk", side_effect=_make_fake_read_chunk(CHUNK)):
@@ -250,7 +250,7 @@ async def test_read_prefetch_triggered_after_first_chunk(manager, ctx):
     file_size = 20 * CHUNK
     prefetch_calls: list = []
 
-    async def fake_prefetch(fs_path, chunks, mtime, ctx):
+    async def fake_prefetch(fs_path, chunks, mtime, gen, ctx):
         prefetch_calls.append(chunks)
 
     with patch.object(manager, "_read_chunk", side_effect=_make_fake_read_chunk(CHUNK)):
@@ -269,7 +269,7 @@ async def test_read_prefetch_stops_at_eof(manager, ctx):
     file_size = 3 * CHUNK  # 3 chunks total (0, 1, 2)
     prefetch_calls: list = []
 
-    async def fake_prefetch(fs_path, chunks, mtime, ctx):
+    async def fake_prefetch(fs_path, chunks, mtime, gen, ctx):
         prefetch_calls.append(chunks)
 
     with patch.object(manager, "_read_chunk", side_effect=_make_fake_read_chunk(CHUNK)):
@@ -283,3 +283,38 @@ async def test_read_prefetch_stops_at_eof(manager, ctx):
     prefetched_chunk_ids = [cid for cid, _ in prefetch_calls[0]]
     assert all(cid < 3 for cid in prefetched_chunk_ids)
     assert 3 not in prefetched_chunk_ids  # chunk 3 doesn't exist
+
+
+# ---------------------------------------------------------------------------
+# Tests: cache generation (invalidate_path)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.trio
+async def test_invalidate_path_bumps_generation(manager):
+    fs = "/cat/sch/vol/f.txt"
+    assert manager._generation(fs) == 0
+    await manager.invalidate_path(fs)
+    assert manager._generation(fs) == 1
+    await manager.invalidate_path(fs)
+    assert manager._generation(fs) == 2
+    # Other paths are unaffected.
+    assert manager._generation("/cat/sch/vol/other.txt") == 0
+
+
+@pytest.mark.trio
+async def test_invalidate_path_changes_the_chunk_cache_key(manager):
+    """After a write bumps the generation, the next lookup keys the chunk under
+    the new generation -- so a stale RAM/disk chunk from before the write (same
+    fs_path, chunk and mtime, but old gen) is bypassed. This is what stops a
+    same-second overwrite (mtime has 1s resolution) from serving stale bytes."""
+    fs = "/cat/sch/vol/f.txt"
+    mtime = 1000.0
+
+    await manager._get_chunk_from_cache_or_disk(fs, 0, mtime, manager._generation(fs))
+    manager.persistence.retrieve_chunk.assert_awaited_with(fs, 0, mtime, 0)
+
+    await manager.invalidate_path(fs)
+
+    await manager._get_chunk_from_cache_or_disk(fs, 0, mtime, manager._generation(fs))
+    manager.persistence.retrieve_chunk.assert_awaited_with(fs, 0, mtime, 1)