fuse4dbricks 0.7.8__tar.gz → 0.7.9__tar.gz

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/CHANGELOG.md

@@ -1,3 +1,21 @@
+# 0.7.9 (2026-10-31)
+
+- Fix prefetch re-queueing chunks that had already been downloaded. A
+  finished prefetch was forgotten, so every following read requested it
+  again until the reader reached it: up to 64 reads per 8 MiB chunk for
+  each chunk in the prefetch window. Each one woke a download worker and
+  checked the disk from a thread. Reading a large file sequentially was
+  CPU-bound, and raising `--prefetch_mb` made it slower. The disk cache
+  now keeps an in-memory index of the chunks it holds, and prefetch skips
+  them.
+- Keep prefetched chunks in the RAM cache as well as on disk, so the read
+  that reaches a chunk no longer reads 8 MiB back from disk first, and
+  concurrent reads of the same chunk no longer each read it back.
+  Enabled by default; `--no-prefetch-to-ram` restores disk-only prefetch.
+- Add `tests/sim_sequential_read.py`, a simulator that drives the real
+  `DataManager` against a bandwidth-limited fake Databricks and disk
+  cache. It reports where time goes in a large sequential read.
+
 # 0.7.8 (2026-10-15)
 
 - Fix typo in DataManager

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: fuse4dbricks
-Version: 0.7.8
+Version: 0.7.9
 Summary: FUSE driver for Databricks Unity Catalog Volumes.
 Project-URL: Homepage, https://github.com/zeehio/fuse4dbricks
 Author-email: Sergio Oller <sergioller@gmail.com>

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/fuse4dbricks/fs/data_manager.py

@@ -116,12 +116,15 @@ class DownloadScheduler:
 
 
 class DataManager:
-    def __init__(self, uc_client: UnityCatalogClient, persistence: DiskPersistence, ram_cache_mb = 512, num_workers=10, prefetch_mb=80):
+    def __init__(self, uc_client: UnityCatalogClient, persistence: DiskPersistence, ram_cache_mb = 512, num_workers=10, prefetch_mb=80, prefetch_to_ram: bool = True):
         """
         uc_client: client to download chunks from UC
         persistence: where to store downloaded chunks on disk
         ram_cache_mb: how much RAM to use for caching chunks in memory (LRU eviction)
         num_workers: how many concurrent downloads to allow
+        prefetch_to_ram: also keep freshly prefetched chunks in the RAM cache,
+            so the read that reaches them does not have to read them back from
+            disk (and concurrent reads of the same chunk do not each do so)
         """
         self.uc_client = uc_client
         self.persistence = persistence
@@ -146,6 +149,7 @@ class DataManager:
         if chunks_to_prefetch < 0:
             chunks_to_prefetch = 0
         self._chunks_to_prefetch = chunks_to_prefetch
+        self._prefetch_to_ram = prefetch_to_ram
 
     def close(self) -> None:
         self._download_scheduler.close()
@@ -181,7 +185,14 @@ class DataManager:
                 # not even be read; a plain existence check is enough to
                 # decide whether a download is needed.
                 if not await self.persistence.chunk_exists(fs_path=chunk_request.fs_path, chunk_index=chunk_request.chunk_id, mtime=chunk_request.mtime, gen=chunk_request.gen):
-                    await self._fetch_chunk(chunk_request)
+                    chunk = await self._fetch_chunk(chunk_request)
+                    # The bytes are already in memory (store_chunk_from_stream
+                    # returns them), so keeping them costs no extra I/O or copy
+                    # and saves the reader an 8 MiB read-back from disk. Chunks
+                    # that were already on disk are NOT loaded: that read-back
+                    # is exactly what the existence check avoids.
+                    if self._prefetch_to_ram:
+                        await self._ram_cache.put(cache_key, chunk)
         finally:
             await self._inflight_coalescer.notify_done(cache_key)
 
@@ -212,6 +223,14 @@ class DataManager:
 
     async def _request_fetch_ahead_chunks(self, fs_path: str, chunks_to_prefetch: list[Tuple[int, int]], mtime: float, gen: int, ctx: pyfuse3.RequestContext):
         for (chunk_id, chunk_size) in chunks_to_prefetch:
+            # Already on disk: nothing to prefetch. Without this check, a chunk
+            # that finished downloading (and so left the coalescer) was
+            # re-enqueued on every read until the reader reached it: 64 reads
+            # per 8 MiB chunk x up to N prefetch chunks, each costing a worker
+            # wake-up and a chunk_exists() thread hop. has_chunk() is a
+            # synchronous in-RAM lookup.
+            if self.persistence.has_chunk(fs_path, chunk_id, mtime, gen):
+                continue
             cache_key = (fs_path, chunk_id, mtime, gen)
             (wait_event, leader) = await self._inflight_coalescer.join_or_lead(cache_key)
             if not leader:

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/fuse4dbricks/main.py

@@ -101,6 +101,14 @@ def parse_args():
         default=80,
         help="Amount of data to prefetch in MB for each file",
     )
+    parser.add_argument(
+        "--prefetch-to-ram",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Keep prefetched chunks in the RAM cache as well as on disk, so "
+        "sequential reads do not read them back from disk "
+        "(--no-prefetch-to-ram: disk only)",
+    )
     parser.add_argument(
         "--num-download-workers",
         type=int,
@@ -254,7 +262,7 @@ async def async_main():
     persistence = DiskPersistence(data_cache_dir, max_size_gb=args.disk_cache_gb, max_age_days=args.disk_cache_max_days)
 
     inode_manager = InodeManager()
-    data_manager = DataManager(uc_client, persistence, ram_cache_mb=args.ram_cache_mb, num_workers=args.num_download_workers, prefetch_mb=args.prefetch_mb)
+    data_manager = DataManager(uc_client, persistence, ram_cache_mb=args.ram_cache_mb, num_workers=args.num_download_workers, prefetch_mb=args.prefetch_mb, prefetch_to_ram=args.prefetch_to_ram)
     metadata_manager = MetadataManager(uc_client, ttl=args.metadata_cache_ttl_sec, max_entries=args.metadata_cache_max_entries, ttl_catalog=args.metadata_cache_ttl_catalog_sec, ttl_negative=args.metadata_cache_ttl_negative_sec, inode_manager=inode_manager)
     # On a 401, the token is invalidated and re-resolved (possibly to a
     # different principal); drop the cached principal so authz re-derives it.

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/fuse4dbricks/storage/persistence.py

@@ -8,6 +8,7 @@ import logging
 import os
 import shutil
 import time
+from functools import lru_cache
 from heapq import heappop, heappush
 from typing import AsyncGenerator, Tuple
 
@@ -16,6 +17,20 @@ import trio
 logger = logging.getLogger(__name__)
 
 
+# access_map value marking an indexed chunk whose file turned out to be missing
+# (deleted behind our back). Real access times are always > 0. The entry is
+# kept, not deleted, so that its size is still released from current_size when
+# evict() later pops the matching access_log record and finds the file gone.
+_MISSING = 0.0
+
+
+@lru_cache(maxsize=4096)
+def _path_hash(fs_path: str) -> str:
+    """sha256 of an fs_path. Cached: has_chunk() runs for every prefetch
+    candidate of every read, and hashing dominated its cost."""
+    return hashlib.sha256(fs_path.encode("utf-8")).hexdigest()
+
+
 def clear_cache(cache_dir: str):
     """Removes *.tmp and *.bin files from cache_dir recursively"""
     logging.info("Clearing cache... %s", cache_dir)
@@ -36,7 +51,9 @@ class DiskPersistence:
         self.current_size = 0
         self.lock = trio.Lock()
         self.access_log: list[Tuple[float, str, int]] = []  # Heap: (now, cache_path, bytes_written)
-        self.access_map: dict[str, float] = {}  # {cache_path: latest_use_timestamp}
+        # {cache_path: latest_use_timestamp}. Also the in-RAM index of which
+        # chunks are on disk (see has_chunk); _MISSING marks a stale entry.
+        self.access_map: dict[str, float] = {}
 
     def run_services(self, nursery):
         """Starts background maintenance and discovery."""
@@ -54,7 +71,7 @@ class DiskPersistence:
         ``_ensure_parent_dir``); a read of a missing chunk simply hits
         FileNotFoundError and returns None.
         """
-        sha256_hash = hashlib.sha256(fs_path.encode("utf-8")).hexdigest()
+        sha256_hash = _path_hash(fs_path)
         shard1 = sha256_hash[:2]
         shard2 = f"{(chunk_index // 1000):07d}"  # Secondary shard to prevent too many files in one dir
         shard_dir = os.path.join(self.cache_dir, shard1, shard2)
@@ -65,6 +82,31 @@ class DiskPersistence:
             shard_dir, f"{sha256_hash}_{mtime_ms}_g{gen}_{chunk_index:07d}.bin"
         )
 
+    def has_chunk(self, fs_path: str, chunk_index: int, mtime: float, gen: int = 0) -> bool:
+        """Whether the chunk is on disk, according to the in-RAM index.
+
+        Synchronous and lock-free (a single dict lookup, no await, no thread
+        hop), so the read path can call it for every prefetch candidate. It
+        does not refresh the LRU timestamp.
+
+        The index can briefly disagree with the disk:
+          * false negatives while the startup scan (``_graceful_init``) is
+            still running, or while ``evict`` removes a file: callers fall back
+            to downloading / ``chunk_exists``, which is harmless;
+          * false positives if a file is deleted behind our back: the next
+            ``retrieve_chunk``/``chunk_exists`` on it notices and marks the
+            entry ``_MISSING``, after which this returns False.
+        """
+        ts = self.access_map.get(self._get_chunk_path(fs_path, chunk_index, mtime, gen))
+        return ts is not None and ts != _MISSING
+
+    async def _mark_missing(self, cache_path: str) -> None:
+        """The index has ``cache_path`` but the file is gone: stop reporting it
+        as present. Size accounting is settled later by evict()."""
+        async with self.lock:
+            if cache_path in self.access_map:
+                self.access_map[cache_path] = _MISSING
+
     @staticmethod
     def _ensure_parent_dir(path: str) -> None:
         """Create the parent directory of ``path`` (idempotent). Runs off the
@@ -130,6 +172,7 @@ class DiskPersistence:
             return data
 
         except FileNotFoundError:
+            await self._mark_missing(cache_path)
             return None
 
     def _read_file(self, path: str) -> bytes:
@@ -149,7 +192,10 @@ class DiskPersistence:
         async with self.lock:
             if cache_path in self.access_map:
                 self.access_map[cache_path] = time.time()
-        return await trio.to_thread.run_sync(os.path.exists, cache_path)
+        exists = await trio.to_thread.run_sync(os.path.exists, cache_path)
+        if not exists:
+            await self._mark_missing(cache_path)
+        return exists
 
     async def store_chunk_from_stream(
         self,

{fuse4dbricks-0.7.8 → fuse4dbricks-0.7.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "fuse4dbricks"
-version = "0.7.8"
+version = "0.7.9"
 description = "FUSE driver for Databricks Unity Catalog Volumes."
 readme = "README.md"
 requires-python = ">=3.11"