gwdatafind-server 2.1.0__tar.gz → 2.2.0__tar.gz

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 2.2.0 - 2025-04-09 { #2.2.0 }
+
+-   Fix directory recursion with Pelican memory cache
+    ([!129](https://git.ligo.org/computing/gwdatafind/server/-/merge_requests/129))
+-   Add Pelican cache versioning and compatibility checks
+    ([!130](https://git.ligo.org/computing/gwdatafind/server/-/merge_requests/130))
+
+[Full details](https://git.ligo.org/computing/gwdatafind/server/-/releases/2.2.0)
+
 ## 2.1.0 - 2025-03-26 { #2.1.0 }
 
 -   Add Pelican database cache file

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gwdatafind-server
-Version: 2.1.0
+Version: 2.2.0
 Summary: The GWDataFind data discover server application
 Author-email: Duncan Macleod <duncan.macleod@ligo.org>, Duncan Meacher <duncan.meacher@ligo.org>
 License: GPL-3.0-or-later

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/docs/changelog.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 2.2.0 - 2025-04-09 { #2.2.0 }
+
+-   Fix directory recursion with Pelican memory cache
+    ([!129](https://git.ligo.org/computing/gwdatafind/server/-/merge_requests/129))
+-   Add Pelican cache versioning and compatibility checks
+    ([!130](https://git.ligo.org/computing/gwdatafind/server/-/merge_requests/130))
+
+[Full details](https://git.ligo.org/computing/gwdatafind/server/-/releases/2.2.0)
+
 ## 2.1.0 - 2025-03-26 { #2.1.0 }
 
 -   Add Pelican database cache file

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/gwdatafind_server/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.1.0'
-__version_tuple__ = version_tuple = (2, 1, 0)
+__version__ = version = '2.2.0'
+__version_tuple__ = version_tuple = (2, 2, 0)
 
-__commit_id__ = commit_id = 'gca848aa98'
+__commit_id__ = commit_id = 'gc287e7f56'

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/gwdatafind_server/source/pelican.py

@@ -22,6 +22,7 @@ from typing import (
     cast,
 )
 
+from .. import __version__ as GWDATAFIND_SERVER_VERSION
 from ..cache import CacheEntry
 from .source import (
     Source,
@@ -67,8 +68,17 @@ class PelicanCache:
     back to SQLite at the end of each update cycle.
     """
 
+    #: Version of the Pelican cache DB schema
     SCHEMA_VERSION = 1
 
+    #: Token to identify compatible cache files
+    #:
+    #: This should be incremented if changes are made to the way the cache is
+    #: constructed or used that would make old cache files incompatible,
+    #: even if the schema version is unchanged.
+    #: This allows the server to force a refresh of the cache if needed.
+    COMPATIBILITY_TOKEN = "pelican-cache-v1"  # noqa: S105
+
     def __init__(self, cache_file: str | Path):
         """Initialize the Pelican cache.
 
@@ -86,9 +96,19 @@ class PelicanCache:
         # Create directory if it doesn't exist
         self.cache_file.parent.mkdir(parents=True, exist_ok=True)
 
+        if self.cache_file.exists():
+            compatible, reason = self._is_existing_cache_compatible()
+            if not compatible:
+                self._invalidate_cache(reason)
+
         with self._get_connection() as conn:
-            # Create tables
-            conn.executescript("""
+            self._create_schema(conn)
+            self._write_metadata(conn)
+            conn.commit()
+
+    def _create_schema(self, conn: sqlite3.Connection) -> None:
+        """Create cache tables and indexes if they don't already exist."""
+        conn.executescript("""
                 CREATE TABLE IF NOT EXISTS directories (
                     path TEXT PRIMARY KEY,
                     last_modified TEXT,
@@ -113,28 +133,72 @@ class PelicanCache:
                     ON urls(directory_path);
                 CREATE INDEX IF NOT EXISTS idx_directories_path
                     ON directories(path);
-            """)
+        """)
+
+    def _read_metadata(self, conn: sqlite3.Connection) -> dict[str, str]:
+        """Read all metadata keys from the database."""
+        cursor = conn.execute("SELECT key, value FROM cache_metadata")
+        return dict(cursor.fetchall())
+
+    def _write_metadata(self, conn: sqlite3.Connection) -> None:
+        """Write metadata required to validate cache compatibility."""
+        conn.executemany(
+            """
+            INSERT INTO cache_metadata (key, value)
+            VALUES (?, ?)
+            ON CONFLICT(key) DO UPDATE SET value=excluded.value
+            """,
+            [
+                ("schema_version", str(self.SCHEMA_VERSION)),
+                ("compatibility_token", self.COMPATIBILITY_TOKEN),
+            ],
+        )
+        conn.execute(
+            "INSERT OR IGNORE INTO cache_metadata (key, value) VALUES (?, ?)",
+            ("created_by", GWDATAFIND_SERVER_VERSION),
+        )
+
+    def _is_existing_cache_compatible(self) -> tuple[bool, str]:
+        """Check whether an existing cache file can be used safely."""
+        try:
+            with self._get_connection() as conn:
+                metadata = self._read_metadata(conn)
+        except sqlite3.Error as exc:
+            return False, f"failed to read metadata: {exc}"
+
+        raw_schema_version = metadata.get("schema_version")
+        if raw_schema_version is None:
+            return False, "missing schema_version metadata"
 
-            # Set/check schema version
-            cursor = conn.execute(
-                "SELECT value FROM cache_metadata WHERE key = ?",
-                ("schema_version",),
+        try:
+            schema_version = int(raw_schema_version)
+        except ValueError:
+            return False, f"invalid schema_version value: {raw_schema_version}"
+
+        if schema_version != self.SCHEMA_VERSION:
+            return False, (
+                "schema_version mismatch "
+                f"(expected={self.SCHEMA_VERSION}, found={schema_version})"
             )
-            result = cursor.fetchone()
 
-            if result is None:
-                # First time setup
-                conn.execute(
-                    "INSERT INTO cache_metadata (key, value) VALUES (?, ?)",
-                    ("schema_version", str(self.SCHEMA_VERSION)),
-                )
-            elif int(result[0]) != self.SCHEMA_VERSION:
-                # Schema migration would go here in future versions
-                self.logger.warning(
-                    "Cache schema version mismatch. Expected %d, got %s",
-                    self.SCHEMA_VERSION,
-                    result[0],
-                )
+        compatibility_token = metadata.get("compatibility_token")
+        if compatibility_token != self.COMPATIBILITY_TOKEN:
+            return False, (
+                "compatibility_token mismatch "
+                f"(expected={self.COMPATIBILITY_TOKEN}, "
+                f"found={compatibility_token})"
+            )
+
+        return True, ""
+
+    def _invalidate_cache(self, reason: str) -> None:
+        """Invalidate an incompatible cache database by deleting it."""
+        self.cache_file.unlink()
+        self.logger.warning(
+            "Invalidated incompatible Pelican cache %s (%s)",
+            self.cache_file,
+            reason,
+        )
 
     @contextmanager
     def _get_connection(self) -> Generator[sqlite3.Connection, None, None]:
@@ -400,6 +464,7 @@ class PelicanSource(Source):
             )
 
         cache: dict[str, list[str]] = defaultdict(list)
+        unchanged_dirs: set[str] = set()
 
         # Walk the tree for URLs
         try:
@@ -416,27 +481,36 @@ class PelicanSource(Source):
             subdirs = cast("dict[str, PelicanInfo]", subdirs)
             files = cast("dict[str, PelicanInfo]", files)
 
-            # parse files
-            for info in files.values():
-                filename = info["name"]
-                url = f"{pelican_url}{filename}"
-                cache[name].append(url)
-                yield url
+            if name in unchanged_dirs:
+                cached_urls = self._cache[name]
+                cache[name] = cached_urls
+                yield from cached_urls
+            else:
+                # Parse files for this directory when not marked unchanged
+                for info in files.values():
+                    filename = info["name"]
+                    url = f"{pelican_url}{filename}"
+                    cache[name].append(url)
+                    yield url
 
-            yield from self._walk_dirnames(subdirs, exclude, cache)
+            self._check_subdirs(subdirs, exclude, unchanged_dirs)
 
         # Update memory cache
         self._cache = cache
 
-    def _walk_dirnames(
+    def _check_subdirs(
         self,
         dirnames: dict[str, PelicanInfo],
         exclude: Iterable[re.Pattern],
-        cache: dict[str, list[str]],
-    ) -> Iterator[str]:
-        """Walk through the dirnames in a Pelican directory walk.
+        unchanged_dirs: set[str],
+    ) -> None:
+        """Check subdirectories in a Pelican directory walk.
+
+        This method attemps to optimise the outer walk() by checking
+        each subdirectory for exclusion patterns, emptiness,
+        and whether it is unchanged based on modification time.
 
-        Note that ``dirnames`` and ``cache`` are modified in place.
+        Note that ``dirnames`` and ``unchanged_dirs`` are modified in place.
         """
         # NOTE: need list() here to get a static copy of the subdirs, so we can
         #       modify the original dict while iterating
@@ -456,25 +530,34 @@ class PelicanSource(Source):
             # if subdir is empty, remove it
             if info["size"] == 0:
                 del dirnames[path]
+                self.logger.debug(
+                    "[%s] Skipping empty subdirectory: %s",
+                    self.name,
+                    name,
+                )
                 continue
 
             mod = info.get("modified")
             # Check if directory is current in memory cache
-            if (
-                name in self._cache
-                and name in self._cache_modified
-                and mod == self._cache_modified[name]  # unchanged
-            ):
-                # Directory is current - yield from cache
-                yield from self._cache[name]
-                cache[name] = self._cache[name]
-                del dirnames[path]
+            if self._is_unchanged(name, mod):
+                unchanged_dirs.add(name)
                 continue
 
             # Store the current modification time for this directory
             if mod is not None:
                 self._cache_modified[name] = mod
 
+    def _is_unchanged(self, name: str, mod: datetime | None) -> bool:
+        """Check if a directory is unchanged based on modification time."""
+        return (
+            # has been cached before
+            name in self._cache
+            # reports its own modification time
+            and mod is not None
+            # and the modification time matches the cached one
+            and self._cache_modified.get(name) == mod
+        )
+
     def _iter_cache(self) -> Iterator[str]:
         for dirurls in self._cache.values():
             yield from dirurls

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/gwdatafind_server.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gwdatafind-server
-Version: 2.1.0
+Version: 2.2.0
 Summary: The GWDataFind data discover server application
 Author-email: Duncan Macleod <duncan.macleod@ligo.org>, Duncan Meacher <duncan.meacher@ligo.org>
 License: GPL-3.0-or-later

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/pyproject.toml

@@ -166,6 +166,11 @@ force-wrap-aliases = true
   "INP001",  # implicit namespace package
 ]
 
+[tool.ruff.lint.pep8-naming]
+extend-ignore-names = [
+  "__version__",  # allow __version__ to be renamed on import
+]
+
 # -- setuptools
 
 [tool.setuptools]

{gwdatafind_server-2.1.0 → gwdatafind_server-2.2.0}/tests/test_source_pelican.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import re
 import sqlite3
+from contextlib import closing
 from datetime import (
     UTC,
     datetime,
@@ -16,6 +17,7 @@ from unittest.mock import patch
 
 import pytest
 
+from gwdatafind_server import __version__ as GWDATAFIND_SERVER_VERSION
 from gwdatafind_server.cache import (
     T010150SourceCache,
 )
@@ -72,6 +74,12 @@ def test_parse_federation_url(input_url: str, expected_output: str) -> None:
 class TestPelicanCache:
     """Test suite for PelicanCache class."""
 
+    @staticmethod
+    def _read_metadata(cache_file: Path) -> dict[str, str]:
+        with closing(sqlite3.connect(cache_file)) as conn:
+            cursor = conn.execute("SELECT key, value FROM cache_metadata")
+            return dict(cursor.fetchall())
+
     @pytest.fixture
     def cache_file(self, tmp_path: Path) -> Path:
         """Temporary cache file path."""
@@ -104,6 +112,17 @@ class TestPelicanCache:
         assert url_cache == {}
         assert modification_times == {}
 
+    def test_init_writes_metadata(self, cache_file: Path):
+        """Test cache initialization writes required metadata keys."""
+        PelicanCache(cache_file)
+        metadata = self._read_metadata(cache_file)
+
+        assert metadata["schema_version"] == str(PelicanCache.SCHEMA_VERSION)
+        assert (
+            metadata["compatibility_token"] == PelicanCache.COMPATIBILITY_TOKEN
+        )
+        assert metadata["created_by"] == GWDATAFIND_SERVER_VERSION
+
     def test_save_and_load_cache(self, pelican_cache: PelicanCache):
         """Test saving and loading cache data."""
         # Prepare test data
@@ -149,6 +168,51 @@ class TestPelicanCache:
         assert loaded_cache == url_cache
         assert loaded_times == {}
 
+    def test_init_invalidates_mismatched_schema_version(self, cache_file: Path):
+        """Test incompatible schema version causes automatic cache invalidation."""
+        cache = PelicanCache(cache_file)
+        cache.save(
+            {"/test/dir": ["pelican://osg-htc.org/gwdata/file.gwf"]},
+            {"/test/dir": datetime.now(UTC)},
+        )
+
+        with closing(sqlite3.connect(cache_file)) as conn:
+            conn.execute(
+                "UPDATE cache_metadata SET value = ? WHERE key = ?",
+                ("0", "schema_version"),
+            )
+            conn.commit()
+
+        reset_cache = PelicanCache(cache_file)
+        loaded_cache, loaded_times = reset_cache.load()
+        metadata = self._read_metadata(cache_file)
+
+        assert loaded_cache == {}
+        assert loaded_times == {}
+        assert metadata["schema_version"] == str(PelicanCache.SCHEMA_VERSION)
+        assert (
+            metadata["compatibility_token"] == PelicanCache.COMPATIBILITY_TOKEN
+        )
+
+    def test_init_invalidates_missing_compatibility_token(self, cache_file: Path):
+        """Test missing compatibility token causes automatic cache invalidation."""
+        PelicanCache(cache_file)
+
+        with closing(sqlite3.connect(cache_file)) as conn:
+            conn.execute(
+                "DELETE FROM cache_metadata WHERE key = ?",
+                ("compatibility_token",),
+            )
+            conn.commit()
+
+        PelicanCache(cache_file)
+        metadata = self._read_metadata(cache_file)
+
+        assert metadata["schema_version"] == str(PelicanCache.SCHEMA_VERSION)
+        assert (
+            metadata["compatibility_token"] == PelicanCache.COMPATIBILITY_TOKEN
+        )
+
     @patch(
         "sqlite3.connect",
         side_effect=sqlite3.OperationalError("mocked failure"),
@@ -272,8 +336,8 @@ class TestPelicanSource(BaseTestSource):
         """Test _exclude_dir with empty patterns list."""
         assert not PelicanSource._exclude_dir("any_file", [])  # noqa: SLF001
 
-    def test_walk_dirnames(self, basic_config: PelicanSourceConfig):
-        """Test _walk_dirnames method."""
+    def test_check_subdirs(self, basic_config: PelicanSourceConfig):
+        """Test _check_subdirs method."""
         source = PelicanSource(basic_config)
 
         # Set up test data
@@ -299,21 +363,21 @@ class TestPelicanSource(BaseTestSource):
         }
 
         exclude_patterns = [re.compile(r".*debug.*")]
-        cache = {}
+        unchanged_dirs: set[str] = set()
 
         # Test the method - don't need to keep return value
-        list(source._walk_dirnames(dirnames, exclude_patterns, cache))  # noqa: SLF001
+        source._check_subdirs(dirnames, exclude_patterns, unchanged_dirs)  # noqa: SLF001
 
         # Verify empty and excluded directories are removed
         assert "/gwdata/O4/tmp" not in dirnames  # Empty directory removed
         assert "/gwdata/O4/debug" not in dirnames  # Excluded directory removed
         assert "/gwdata/O4/strain" in dirnames  # Valid directory remains
 
-    def test_walk_dirnames_with_cache_hit(
+    def test_check_subdirs_with_cache_hit(
         self,
         basic_config: PelicanSourceConfig,
     ):
-        """Test _walk_dirnames with cache hit."""
+        """Test _check_subdirs marks unchanged directories for cache use."""
         source = PelicanSource(basic_config)
 
         # Set up existing cache
@@ -335,16 +399,84 @@ class TestPelicanSource(BaseTestSource):
             },
         }
 
-        cache = {}
+        unchanged_dirs: set[str] = set()
 
         # Test the method
-        urls = list(source._walk_dirnames(dirnames, [], cache))  # noqa: SLF001
+        source._check_subdirs(dirnames, [], unchanged_dirs)  # noqa: SLF001
+
+        # Verify directory is marked unchanged and retained for traversal
+        assert "/gwdata/O4/strain" in unchanged_dirs
+        assert "/gwdata/O4/strain" in dirnames
+
+    def test_query_pelican_uses_cache_without_skipping_descendants(
+        self,
+        basic_config: PelicanSourceConfig,
+    ):
+        """Test unchanged directories use cache while descendants are still walked."""
+        source = PelicanSource(basic_config)
+
+        mod_time = datetime(2024, 1, 1, tzinfo=UTC)
+        new_mod = datetime(2024, 1, 2, tzinfo=UTC)
+        parent_cached_url = "pelican://osg-htc.org/gwdata/O4/parent.gwf"
+        source._cache = {  # noqa: SLF001
+            "/gwdata/O4": [parent_cached_url],
+        }
+        source._cache_modified = {  # noqa: SLF001
+            "/gwdata/O4": mod_time,
+        }
+
+        parent_subdirs: dict[str, PelicanInfo] = {
+            "/gwdata/O4/new": {
+                "name": "/gwdata/O4/new/",
+                "size": 1024,
+                "type": "directory",
+                "modified": new_mod,
+            },
+        }
+        parent_files: dict[str, PelicanInfo] = {
+            "parent.gwf": {
+                "name": "/gwdata/O4/parent.gwf",
+                "size": 128,
+                "type": "file",
+                "modified": mod_time,
+            },
+        }
+        child_files: dict[str, PelicanInfo] = {
+            "child.gwf": {
+                "name": "/gwdata/O4/new/child.gwf",
+                "size": 256,
+                "type": "file",
+                "modified": new_mod,
+            },
+        }
+
+        walk_entries = [
+            ("/gwdata/O4", parent_subdirs, parent_files),
+            ("/gwdata/O4/new", {}, child_files),
+        ]
+
+        with patch("pelicanfs.core.PelicanFileSystem") as mock_pelfs_init:
+            mock_pelfs = mock_pelfs_init.return_value
+            mock_pelfs.discovery_url = "pelican://osg-htc.org"
+            mock_pelfs.dircache.use_listings_cache = True
+            mock_pelfs.http_file_system.dircache.use_listings_cache = True
+            mock_pelfs.fastwalk.return_value = iter(walk_entries)
+
+            urls = list(
+                source.query_pelican(
+                    source.federation_url,
+                    source.path,
+                ),
+            )
+
+        child_url = "pelican://osg-htc.org/gwdata/O4/new/child.gwf"
+
+        # Parent comes from cache exactly once; child is discovered by traversal
+        assert urls == [parent_cached_url, child_url]
 
-        # Verify cache was used
-        assert urls == ["cached_url1", "cached_url2"]
-        # No trailing slash in cache key
-        assert cache["/gwdata/O4/strain"] == ["cached_url1", "cached_url2"]
-        assert "/gwdata/O4/strain" not in dirnames  # Directory removed after cache hit
+        # Cache is refreshed with both parent and child directories
+        assert source._cache["/gwdata/O4"] == [parent_cached_url]  # noqa: SLF001
+        assert source._cache["/gwdata/O4/new"] == [child_url]  # noqa: SLF001
 
     def test_run_method_basic(
         self,