antsibull-nox 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

antsibull_nox/collection/install.py

@@ -20,6 +20,7 @@ from pathlib import Path
 
 from antsibull_fileutils.yaml import load_yaml_file
 
+from ..ansible import AnsibleCoreVersion
 from ..paths import copy_collection as _paths_copy_collection
 from ..paths import remove_path as _remove
 from .data import CollectionData, CollectionSource, SetupResult
@@ -47,84 +48,170 @@ class _CollectionSources:
         """
         self.sources[name] = source
 
-    def get_source(self, name: str) -> CollectionSource:
+    @t.overload
+    def get_source(
+        self, name: str, *, create_default: t.Literal[True] = True
+    ) -> CollectionSource: ...
+
+    @t.overload
+    def get_source(
+        self, name: str, *, create_default: t.Literal[False]
+    ) -> CollectionSource | None: ...
+
+    def get_source(
+        self, name: str, *, create_default: bool = True
+    ) -> CollectionSource | None:
         """
         Get source for collection.
         """
         source = self.sources.get(name)
-        if source is None:
+        if source is None and create_default:
             source = CollectionSource(name, name)
         return source
 
 
+class _CollectionDownloadCache:
+    @staticmethod
+    def _parse_galaxy_filename(file: Path) -> tuple[str, str, str]:
+        """
+        Split filename into (namespace, name, version) tuple.
+        """
+        if not file.name.endswith(_TARBALL_EXTENSION):
+            raise ValueError(
+                f"Filename {file.name!r} does not end with {_TARBALL_EXTENSION}"
+            )
+        parts = file.name[: -len(_TARBALL_EXTENSION)].split("-", 2)
+        if len(parts) != 3:
+            raise ValueError(
+                f"Filename {file.name!r} does not belong to a Galaxy tarball"
+            )
+        return parts[0], parts[1], parts[2]
+
+    @staticmethod
+    def _parse_cache_filename(file: Path) -> tuple[str, str, str, str] | None:
+        """
+        Split cache filename into (namespace, name, source_id, version) tuple.
+        """
+        if not file.name.endswith(_TARBALL_EXTENSION):
+            return None
+        parts = file.name[: -len(_TARBALL_EXTENSION)].split("-", 3)
+        if len(parts) != 4:
+            return None
+        return parts[0], parts[1], parts[2], parts[3]
+
+    @staticmethod
+    def _encode_cache_filename(
+        namespace: str, name: str, version: str, source: CollectionSource
+    ) -> str:
+        return f"{namespace}-{name}-{source.identifier()}-{version}{_TARBALL_EXTENSION}"
+
+    def download_collections(
+        self, *, destination: Path, sources: list[CollectionSource], runner: Runner
+    ) -> None:
+        """
+        Given a set of collection sources, downloads these and stores them in destination.
+        """
+        destination.mkdir(exist_ok=True)
+        names = ", ".join(sorted(source.name for source in sources))
+        print(f"Downloading {names} to {destination}...")
+        sources_by_name = {}
+        for source in sources:
+            sources_by_name[source.name] = source
+            if source.name != source.source:
+                print(f"  Installing {source.name} via {source.source}...")
+        with tempfile.TemporaryDirectory(
+            prefix="antsibull-nox-galaxy-download"
+        ) as dest:
+            tempdir = Path(dest)
+            command = [
+                "ansible-galaxy",
+                "collection",
+                "download",
+                "--no-deps",
+                "--download-path",
+                str(tempdir),
+                "--",
+                *(source.source for source in sources),
+            ]
+            runner(command)
+            for file in tempdir.iterdir():
+                if file.name.endswith(_TARBALL_EXTENSION) and file.is_file():
+                    namespace, name, version = self._parse_galaxy_filename(file)
+                    source_opt = sources_by_name.get(f"{namespace}.{name}")
+                    if source_opt is None:
+                        print(
+                            f"Found unknown collection artifact {file.name!r}, ignoring..."
+                        )
+                        continue
+                    destfile = destination / self._encode_cache_filename(
+                        namespace, name, version, source_opt
+                    )
+                    _remove(destfile)
+                    shutil.move(file, destfile)
+
+    def list_downloaded_dir(
+        self, *, path: Path
+    ) -> dict[tuple[str, str], tuple[Path, str]]:
+        """
+        List contents of download cache.
+
+        Returns a dictionary mapping (collection_name, source_id) tuples to
+        (tarball_path, version) tuples.
+        """
+        if not path.is_dir():
+            return {}
+        result: dict[tuple[str, str], tuple[Path, str]] = {}
+        for file in path.iterdir():
+            if not file.is_file():
+                continue
+            parsed = self._parse_cache_filename(file)
+            if parsed is None:
+                continue
+            namespace, name, source_id, version = parsed
+            collection_name = f"{namespace}.{name}"
+            key = collection_name, source_id
+            if key in result:
+                # Determine older entry
+                old_file = result[key][0]
+                old_stat = old_file.stat()
+                new_stat = file.stat()
+                if new_stat.st_mtime > old_stat.st_mtime:
+                    older_file = old_file
+                    result[key] = file, version
+                else:
+                    older_file = file
+                # Clean up older entry
+                _remove(older_file)
+            else:
+                result[key] = file, version
+        return result
+
+
 _COLLECTION_SOURCES = _CollectionSources()
+_COLLECTION_SOURCES_PER_CORE_VERSION: dict[AnsibleCoreVersion, _CollectionSources] = {}
+_COLLECTION_DOWNLOAD_CACHE = _CollectionDownloadCache()
 _TARBALL_EXTENSION = ".tar.gz"
 _INSTALLATION_CONFIG_ENV_VAR = "ANTSIBULL_NOX_INSTALL_COLLECTIONS"
 
 
-def setup_collection_sources(collection_sources: dict[str, CollectionSource]) -> None:
+def setup_collection_sources(
+    collection_sources: dict[str, CollectionSource],
+    *,
+    ansible_core_version: AnsibleCoreVersion | None = None,
+) -> None:
     """
     Setup collection sources.
     """
+    sources = _COLLECTION_SOURCES
+    if ansible_core_version is not None:
+        sources = _COLLECTION_SOURCES_PER_CORE_VERSION.get(
+            ansible_core_version, _COLLECTION_SOURCES
+        )
+        if sources is _COLLECTION_SOURCES:
+            sources = _CollectionSources()
+            _COLLECTION_SOURCES_PER_CORE_VERSION[ansible_core_version] = sources
     for name, source in collection_sources.items():
-        _COLLECTION_SOURCES.set_source(name, source)
-
-
-def _download_collections(
-    *, destination: Path, sources: list[CollectionSource], runner: Runner
-) -> None:
-    destination.mkdir(exist_ok=True)
-    names = ", ".join(sorted(source.name for source in sources))
-    print(f"Downloading {names} to {destination}...")
-    for source in sources:
-        if source.name != source.source:
-            print(f"  Installing {source.name} via {source.source}...")
-    with tempfile.TemporaryDirectory(prefix="antsibull-nox-galaxy-download") as dest:
-        tempdir = Path(dest)
-        command = [
-            "ansible-galaxy",
-            "collection",
-            "download",
-            "--no-deps",
-            "--download-path",
-            str(tempdir),
-            "--",
-            *(source.source for source in sources),
-        ]
-        runner(command)
-        for file in tempdir.iterdir():
-            if file.name.endswith(_TARBALL_EXTENSION) and file.is_file():
-                destfile = destination / file.name
-                _remove(destfile)
-                shutil.move(file, destfile)
-
-
-def _list_downloaded_dir(*, path: Path) -> dict[str, Path]:
-    if not path.is_dir():
-        return {}
-    result: dict[str, Path] = {}
-    for file in path.iterdir():
-        if not file.name.endswith(_TARBALL_EXTENSION) or not file.is_file():
-            continue
-        basename = file.name[: -len(_TARBALL_EXTENSION)]
-        # Format: community-internal_test_tools-0.15.0, community-aws-10.0.0-dev0
-        parts = basename.split("-", 2)
-        if len(parts) != 3:
-            continue
-        full_name = ".".join(parts[:2])
-        if full_name in result:
-            old_stat = result[full_name].stat()
-            new_stat = file.stat()
-            if new_stat.st_mtime > old_stat.st_mtime:
-                older_file = result[full_name]
-                result[full_name] = file
-            else:
-                older_file = file
-            # Clean up older entry
-            _remove(older_file)
-        else:
-            result[full_name] = file
-    return result
+        sources.set_source(name, source)
 
 
 def _install_from_download_cache(
@@ -137,9 +224,21 @@ def _install_from_download_cache(
     return destination_dir
 
 
+def _get_source(
+    name: str, *, ansible_core_version: AnsibleCoreVersion
+) -> CollectionSource:
+    sources_per_version = _COLLECTION_SOURCES_PER_CORE_VERSION.get(ansible_core_version)
+    if sources_per_version:
+        result = sources_per_version.get_source(name, create_default=False)
+        if result is not None:
+            return result
+    return _COLLECTION_SOURCES.get_source(name)
+
+
 def _install_missing(
     collections: list[str],
     *,
+    ansible_core_version: AnsibleCoreVersion,
     runner: Runner,
 ) -> list[CollectionData]:
     config = os.environ.get(_INSTALLATION_CONFIG_ENV_VAR)
@@ -151,38 +250,49 @@ def _install_missing(
             f" thus cannot install missing exception{plural_s} {names}..."
         )
         return []
-    sources = [_COLLECTION_SOURCES.get_source(name) for name in collections]
+    sources = [
+        _get_source(name, ansible_core_version=ansible_core_version)
+        for name in collections
+    ]
     result: list[CollectionData] = []
-    with _update_collection_list() as updater:
+    with _update_collection_list(ansible_core_version=ansible_core_version) as updater:
         global_cache = updater.get_global_cache()
-        install: list[str] = []
+        install: list[CollectionSource] = []
         download: list[CollectionSource] = []
-        download_cache = _list_downloaded_dir(path=global_cache.download_cache)
+        download_cache = _COLLECTION_DOWNLOAD_CACHE.list_downloaded_dir(
+            path=global_cache.download_cache
+        )
         for source in sources:
             if cd := updater.find(source.name):
                 result.append(cd)
             else:
-                install.append(source.name)
-                if not download_cache.get(source.name):
+                install.append(source)
+                if not download_cache.get((source.name, source.identifier())):
                     download.append(source)
         if download:
-            _download_collections(
+            _COLLECTION_DOWNLOAD_CACHE.download_collections(
                 destination=global_cache.download_cache, sources=download, runner=runner
             )
-            download_cache = _list_downloaded_dir(path=global_cache.download_cache)
+            download_cache = _COLLECTION_DOWNLOAD_CACHE.list_downloaded_dir(
+                path=global_cache.download_cache
+            )
         if install:
-            for name in install:
-                if name not in download_cache:
+            for source in install:
+                key = source.name, source.identifier()
+                if key not in download_cache:
                     raise ValueError(
-                        f"Error: cannot find {name} in download cache"
-                        f" {global_cache.download_cache} after successful download!"
+                        f"Error: cannot find {source.name} (source ID {source.identifier()})"
+                        f" in download cache {global_cache.download_cache}"
+                        " after successful download!"
                     )
                 c_dir = _install_from_download_cache(
-                    full_name=name,
-                    tarball=download_cache[name],
-                    destination=global_cache.extracted_cache,
+                    full_name=source.name,
+                    tarball=download_cache[key][0],
+                    destination=global_cache.get_extracted_path(
+                        ansible_core_version=ansible_core_version
+                    ),
                 )
-                c_namespace, c_name = name.split(".", 1)
+                c_namespace, c_name = source.name.split(".", 1)
                 result.append(
                     updater.add_collection(
                         directory=c_dir, namespace=c_namespace, name=c_name
@@ -414,6 +524,7 @@ def setup_collections(
     destination: str | os.PathLike,
     runner: Runner,
     *,
+    ansible_core_version: AnsibleCoreVersion,
     extra_collections: list[str] | None = None,
     extra_deps_files: list[str | os.PathLike] | None = None,
     global_cache_dir: Path,
@@ -423,7 +534,9 @@ def setup_collections(
     Setup all collections in a tree structure inside the destination directory.
     """
     all_collections = get_collection_list(
-        runner=runner, global_cache_dir=global_cache_dir
+        runner=runner,
+        global_cache_dir=global_cache_dir,
+        ansible_core_version=ansible_core_version,
     )
     destination_root = Path(destination) / "ansible_collections"
     destination_root.mkdir(exist_ok=True)
@@ -451,7 +564,9 @@ def setup_collections(
         if missing.is_empty():
             break
         for collection_data in _install_missing(
-            missing.get_missing_names(), runner=runner
+            missing.get_missing_names(),
+            ansible_core_version=ansible_core_version,
+            runner=runner,
         ):
             collections_to_install[collection_data.full_name] = collection_data
             missing.remove(collection_data.full_name)

antsibull_nox/collection/search.py

@@ -21,6 +21,7 @@ from pathlib import Path
 
 from antsibull_fileutils.yaml import load_yaml_file
 
+from ..ansible import AnsibleCoreVersion
 from .data import CollectionData
 
 # Function that runs a command (and fails on non-zero return code)
@@ -28,6 +29,10 @@ from .data import CollectionData
 Runner = t.Callable[[list[str]], tuple[bytes, bytes]]
 
 
+GALAXY_YML = "galaxy.yml"
+MANIFEST_JSON = "MANIFEST.json"
+
+
 @dataclass(frozen=True)
 class _GlobalCache:
     root: Path
@@ -45,6 +50,12 @@ class _GlobalCache:
             extracted_cache=root / "extracted",
         )
 
+    def get_extracted_path(self, *, ansible_core_version: AnsibleCoreVersion) -> Path:
+        """
+        Given an ansible-core version, returns its extracted collection cache directory.
+        """
+        return self.extracted_cache / str(ansible_core_version)
+
 
 def _load_galaxy_yml(galaxy_yml: Path) -> dict[str, t.Any]:
     try:
@@ -80,19 +91,19 @@ def load_collection_data_from_disk(
     """
     Load collection data from disk.
     """
-    galaxy_yml = path / "galaxy.yml"
-    manifest_json = path / "MANIFEST.json"
+    galaxy_yml = path / GALAXY_YML
+    manifest_json = path / MANIFEST_JSON
     found: Path
     if galaxy_yml.is_file():
         found = galaxy_yml
         data = _load_galaxy_yml(galaxy_yml)
     elif not accept_manifest:
-        raise ValueError(f"Cannot find galaxy.yml in {path}")
+        raise ValueError(f"Cannot find {GALAXY_YML} in {path}")
     elif manifest_json.is_file():
         found = manifest_json
         data = _load_manifest_json_collection_info(manifest_json)
     else:
-        raise ValueError(f"Cannot find galaxy.yml or MANIFEST.json in {path}")
+        raise ValueError(f"Cannot find {GALAXY_YML} or {MANIFEST_JSON} in {path}")
 
     ns = data.get("namespace")
     if not isinstance(ns, str):
@@ -275,9 +286,9 @@ class CollectionList:
         )
 
     @classmethod
-    def collect(cls, *, runner: Runner, global_cache: _GlobalCache) -> CollectionList:
+    def collect_global(cls, *, runner: Runner) -> CollectionList:
         """
-        Search for a list of collections. The result is not cached.
+        Search for a global list of collections. The result is not cached.
         """
         found_collections = {}
         for collection_data in _fs_list_local_collections():
@@ -287,7 +298,25 @@ class CollectionList:
                 # Similar to Ansible, we use the first match
                 if collection_data.full_name not in found_collections:
                     found_collections[collection_data.full_name] = collection_data
-        for collection_data in _fs_list_global_cache(global_cache.extracted_cache):
+        return cls.create(found_collections)
+
+    @classmethod
+    def collect_local(
+        cls,
+        *,
+        ansible_core_version: AnsibleCoreVersion,
+        global_cache: _GlobalCache,
+        current: CollectionData,
+    ) -> CollectionList:
+        """
+        Search for a list of collections from a local cache path. The result is not cached.
+        """
+        found_collections = {
+            current.full_name: current,
+        }
+        for collection_data in _fs_list_global_cache(
+            global_cache.get_extracted_path(ansible_core_version=ansible_core_version)
+        ):
             # Similar to Ansible, we use the first match
             if collection_data.full_name not in found_collections:
                 found_collections[collection_data.full_name] = collection_data
@@ -309,6 +338,18 @@ class CollectionList:
             current=self.current,
         )
 
+    def merge_with(self, other: CollectionList) -> CollectionList:
+        """
+        Merge this collection list with another (local) one.
+        """
+        result = dict(self.collection_map)
+        for collection in other.collections:
+            # Similar to Ansible, we use the first match.
+            # For merge, this means we prefer self over other.
+            if collection.full_name not in result:
+                result[collection.full_name] = collection
+        return CollectionList.create(result)
+
     def _add(self, collection: CollectionData, *, force: bool = True) -> bool:
         if not force and collection.full_name in self.collection_map:
             return False
@@ -319,16 +360,23 @@ class CollectionList:
 
 class _CollectionListUpdater:
     def __init__(
-        self, *, owner: "_CollectionListSingleton", collection_list: CollectionList
+        self,
+        *,
+        owner: "_CollectionListSingleton",
+        merged_collection_list: CollectionList,
+        local_collection_list: CollectionList,
+        ansible_core_version: AnsibleCoreVersion,
     ) -> None:
         self._owner = owner
-        self._collection_list = collection_list
+        self._merged_collection_list = merged_collection_list
+        self._local_collection_list = local_collection_list
+        self._ansible_core_version = ansible_core_version
 
     def find(self, name: str) -> CollectionData | None:
         """
         Find a collection for a given name.
         """
-        return self._collection_list.find(name)
+        return self._merged_collection_list.find(name)
 
     def add_collection(
         self, *, directory: Path, namespace: str, name: str
@@ -337,9 +385,15 @@ class _CollectionListUpdater:
         Add a new collection to the cache.
         """
         # pylint: disable-next=protected-access
-        return self._owner._add_collection(
-            directory=directory, namespace=namespace, name=name
+        result = self._owner._add_collection(
+            directory=directory,
+            namespace=namespace,
+            name=name,
+            ansible_core_version=self._ansible_core_version,
         )
+        self._merged_collection_list._add(result)  # pylint: disable=protected-access
+        self._local_collection_list._add(result)  # pylint: disable=protected-access
+        return result
 
     def get_global_cache(self) -> _GlobalCache:
         """
@@ -352,7 +406,10 @@ class _CollectionListSingleton:
     _lock = threading.Lock()
 
     _global_cache_dir: Path | None = None
-    _collection_list: CollectionList | None = None
+    _global_collection_list: CollectionList | None = None
+    _global_collection_list_per_ansible_core_version: dict[
+        AnsibleCoreVersion, CollectionList
+    ] = {}
 
     def setup(self, *, global_cache_dir: Path) -> None:
         """
@@ -374,30 +431,48 @@ class _CollectionListSingleton:
         Clear collection cache.
         """
         with self._lock:
-            self._collection_list = None
+            self._global_collection_list = None
+            self._global_collection_list_per_ansible_core_version.clear()
 
-    def get_cached(self) -> CollectionList | None:
+    def get_cached(
+        self, *, ansible_core_version: AnsibleCoreVersion | None = None
+    ) -> CollectionList | None:
         """
         Return cached list of collections, if present.
         Do not modify the result!
         """
-        return self._collection_list
+        if ansible_core_version is None:
+            return self._global_collection_list
+        return self._global_collection_list_per_ansible_core_version.get(
+            ansible_core_version
+        )
 
-    def get(self, *, runner: Runner) -> CollectionList:
+    def get(
+        self, *, ansible_core_version: AnsibleCoreVersion, runner: Runner
+    ) -> CollectionList:
         """
         Search for a list of collections. The result is cached.
         """
         with self._lock:
             if self._global_cache_dir is None:
                 raise ValueError("Internal error: global cache dir not setup")
-            result = self._collection_list
-            if result is None:
-                result = CollectionList.collect(
-                    runner=runner,
+            global_list = self._global_collection_list
+            if global_list is None:
+                global_list = CollectionList.collect_global(runner=runner)
+                self._global_collection_list = global_list
+            local_list = self._global_collection_list_per_ansible_core_version.get(
+                ansible_core_version
+            )
+            if local_list is None:
+                local_list = CollectionList.collect_local(
                     global_cache=_GlobalCache.create(root=self._global_cache_dir),
+                    ansible_core_version=ansible_core_version,
+                    current=global_list.current,
                 )
-                self._collection_list = result
-        return result.clone()
+                self._global_collection_list_per_ansible_core_version[
+                    ansible_core_version
+                ] = local_list
+        return global_list.merge_with(local_list)
 
     def _get_global_cache(self) -> _GlobalCache:
         """
@@ -408,26 +483,45 @@ class _CollectionListSingleton:
         return _GlobalCache.create(root=self._global_cache_dir)
 
     def _add_collection(
-        self, *, directory: Path, namespace: str, name: str
+        self,
+        *,
+        directory: Path,
+        namespace: str,
+        name: str,
+        ansible_core_version: AnsibleCoreVersion,
     ) -> CollectionData:
         """
         Add collection in directory if the collection list has been cached.
         """
-        if not self._collection_list:
-            raise ValueError("Internal error: collections not listed")
+        local_list = self._global_collection_list_per_ansible_core_version.get(
+            ansible_core_version
+        )
+        if not local_list:
+            raise ValueError(
+                f"Internal error: collections not listed for {ansible_core_version}"
+            )
         data = load_collection_data_from_disk(directory, namespace=namespace, name=name)
-        self._collection_list._add(data)  # pylint: disable=protected-access
+        local_list._add(data)  # pylint: disable=protected-access
         return data
 
     @contextmanager
-    def _update_collection_list(self) -> t.Iterator[_CollectionListUpdater]:
+    def _update_collection_list(
+        self, *, ansible_core_version: AnsibleCoreVersion
+    ) -> t.Iterator[_CollectionListUpdater]:
         with self._lock:
-            if not self._collection_list or self._global_cache_dir is None:
+            global_list = self._global_collection_list
+            local_list = self._global_collection_list_per_ansible_core_version.get(
+                ansible_core_version
+            )
+            if not global_list or self._global_cache_dir is None or local_list is None:
                 raise ValueError(
                     "Internal error: collections not listed or global cache not setup"
                 )
             yield _CollectionListUpdater(
-                owner=self, collection_list=self._collection_list
+                owner=self,
+                merged_collection_list=global_list.merge_with(local_list),
+                local_collection_list=local_list,
+                ansible_core_version=ansible_core_version,
             )
 
 
@@ -435,18 +529,26 @@ _COLLECTION_LIST = _CollectionListSingleton()
 
 
 @contextmanager
-def _update_collection_list() -> t.Iterator[_CollectionListUpdater]:
+def _update_collection_list(
+    *, ansible_core_version: AnsibleCoreVersion
+) -> t.Iterator[_CollectionListUpdater]:
     # pylint: disable-next=protected-access
-    with _COLLECTION_LIST._update_collection_list() as result:
+    with _COLLECTION_LIST._update_collection_list(
+        ansible_core_version=ansible_core_version
+    ) as result:
         yield result
 
 
-def get_collection_list(*, runner: Runner, global_cache_dir: Path) -> CollectionList:
+def get_collection_list(
+    *, runner: Runner, global_cache_dir: Path, ansible_core_version: AnsibleCoreVersion
+) -> CollectionList:
     """
     Search for a list of collections. The result is cached.
     """
     _COLLECTION_LIST.setup(global_cache_dir=global_cache_dir)
-    return _COLLECTION_LIST.get(runner=runner)
+    return _COLLECTION_LIST.get(
+        runner=runner, ansible_core_version=ansible_core_version
+    )
 
 
 __all__ = [