antsibull-nox 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

antsibull_nox/{collection.py → collection/search.py}

@@ -10,63 +10,39 @@ Handle Ansible collections.
 
 from __future__ import annotations
 
-import functools
 import json
 import os
+import threading
 import typing as t
-from collections.abc import Collection, Iterable, Iterator, Sequence
+from collections.abc import Collection, Iterator, Sequence
+from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
 
-from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file
 
-from .paths import copy_collection as _paths_copy_collection
-from .paths import remove_path as _remove
+from .data import CollectionData
 
 # Function that runs a command (and fails on non-zero return code)
 # and returns a tuple (stdout, stderr)
 Runner = t.Callable[[list[str]], tuple[bytes, bytes]]
 
 
-@dataclass
-class CollectionData:  # pylint: disable=too-many-instance-attributes
-    """
-    An Ansible collection.
-    """
-
-    collections_root_path: Path | None
-    path: Path
-    namespace: str
-    name: str
-    full_name: str
-    version: str | None
-    dependencies: dict[str, str]
-    current: bool
+@dataclass(frozen=True)
+class _GlobalCache:
+    root: Path
+    download_cache: Path
+    extracted_cache: Path
 
     @classmethod
-    def create(
-        cls,
-        *,
-        collections_root_path: Path | None = None,
-        path: Path,
-        full_name: str,
-        version: str | None = None,
-        dependencies: dict[str, str] | None = None,
-        current: bool = False,
-    ):
+    def create(cls, *, root: Path) -> _GlobalCache:
         """
-        Create a CollectionData object.
+        Create a global cache object.
         """
-        namespace, name = full_name.split(".", 1)
-        return CollectionData(
-            collections_root_path=collections_root_path,
-            path=path,
-            namespace=namespace,
-            name=name,
-            full_name=full_name,
-            version=version,
-            dependencies=dependencies or {},
-            current=current,
+        return cls(
+            root=root,
+            download_cache=root / "downloaded",
+            extracted_cache=root / "extracted",
         )
 
 
@@ -149,25 +125,6 @@ def load_collection_data_from_disk(
     )
 
 
-def force_collection_version(path: Path, *, version: str) -> bool:
-    """
-    Make sure galaxy.yml contains this version.
-
-    Returns ``True`` if the version was changed, and ``False`` if the version
-    was already set to this value.
-    """
-    galaxy_yml = path / "galaxy.yml"
-    try:
-        data = load_yaml_file(galaxy_yml)
-    except Exception as exc:
-        raise ValueError(f"Cannot parse {galaxy_yml}: {exc}") from exc
-    if data.get("version") == version:
-        return False
-    data["version"] = version
-    store_yaml_file(galaxy_yml, data)
-    return True
-
-
 def _list_adjacent_collections_ansible_collections_tree(
     root: Path,
     *,
@@ -189,10 +146,11 @@ def _list_adjacent_collections_ansible_collections_tree(
                                 root=root,
                             )
                     except Exception:  # pylint: disable=broad-exception-caught
-                        # If name doesn't happen to be a (symlink to a) directory, ...
+                        # If name doesn't happen to be a (symlink to a) directory,
+                        # is not readable, ...
                         pass
         except Exception:  # pylint: disable=broad-exception-caught
-            # If namespace doesn't happen to be a (symlink to a) directory, ...
+            # If namespace doesn't happen to be a (symlink to a) directory, is not readable, ...
             pass
 
 
@@ -256,6 +214,16 @@ def _fs_list_local_collections() -> Iterator[CollectionData]:
         yield from _list_adjacent_collections_outside_tree(
             parents[0], directories_to_ignore=(cwd,)
         )
+    else:
+        # Only happens if cwd == "/"
+        pass  # pragma: no cover
+
+
+def _fs_list_global_cache(global_cache_dir: Path) -> Iterator[CollectionData]:
+    if not global_cache_dir.is_dir():
+        return
+
+    yield from _list_adjacent_collections_outside_tree(global_cache_dir)
 
 
 def _galaxy_list_collections(runner: Runner) -> Iterator[CollectionData]:
@@ -307,14 +275,19 @@ class CollectionList:
         )
 
     @classmethod
-    def collect(cls, runner: Runner) -> CollectionList:
+    def collect(cls, *, runner: Runner, global_cache: _GlobalCache) -> CollectionList:
         """
         Search for a list of collections. The result is not cached.
         """
         found_collections = {}
         for collection_data in _fs_list_local_collections():
             found_collections[collection_data.full_name] = collection_data
-        for collection_data in _galaxy_list_collections(runner):
+        if os.environ.get("ANTSIBULL_NOX_IGNORE_INSTALLED_COLLECTIONS") != "true":
+            for collection_data in _galaxy_list_collections(runner):
+                # Similar to Ansible, we use the first match
+                if collection_data.full_name not in found_collections:
+                    found_collections[collection_data.full_name] = collection_data
+        for collection_data in _fs_list_global_cache(global_cache.extracted_cache):
             # Similar to Ansible, we use the first match
             if collection_data.full_name not in found_collections:
                 found_collections[collection_data.full_name] = collection_data
@@ -326,220 +299,158 @@ class CollectionList:
         """
         return self.collection_map.get(name)
 
+    def clone(self) -> CollectionList:
+        """
+        Create a clone of this list.
+        """
+        return CollectionList(
+            collections=list(self.collections),
+            collection_map=dict(self.collection_map),
+            current=self.current,
+        )
 
-@functools.cache
-def get_collection_list(runner: Runner) -> CollectionList:
-    """
-    Search for a list of collections. The result is cached.
-    """
-    return CollectionList.collect(runner)
-
-
-def _add_all_dependencies(
-    collections: dict[str, CollectionData], all_collections: CollectionList
-) -> None:
-    to_process = list(collections.values())
-    while to_process:
-        collection = to_process.pop(0)
-        for dependency_name in collection.dependencies:
-            if dependency_name not in collections:
-                dependency_data = all_collections.find(dependency_name)
-                if dependency_data is None:
-                    raise ValueError(
-                        f"Cannot find collection {dependency_name},"
-                        f" a dependency of {collection.full_name}!"
-                    )
-                collections[dependency_name] = dependency_data
-                to_process.append(dependency_data)
+    def _add(self, collection: CollectionData, *, force: bool = True) -> bool:
+        if not force and collection.full_name in self.collection_map:
+            return False
+        self.collections.append(collection)
+        self.collection_map[collection.full_name] = collection
+        return True
 
 
-def _install_collection(collection: CollectionData, path: Path) -> None:
-    if path.is_symlink():
-        if path.readlink() == collection.path:
-            return
-        path.unlink()
-    else:
-        _remove(path)
-    path.symlink_to(collection.path)
+class _CollectionListUpdater:
+    def __init__(
+        self, *, owner: "_CollectionListSingleton", collection_list: CollectionList
+    ) -> None:
+        self._owner = owner
+        self._collection_list = collection_list
 
+    def find(self, name: str) -> CollectionData | None:
+        """
+        Find a collection for a given name.
+        """
+        return self._collection_list.find(name)
 
-def _install_current_collection(collection: CollectionData, path: Path) -> None:
-    if path.exists() and (path.is_symlink() or not path.is_dir()):
-        path.unlink()
-    path.mkdir(exist_ok=True)
-    present = {p.name for p in path.iterdir()}
-    for source_entry in collection.path.iterdir():
-        if source_entry.name == ".nox":
-            continue
-        dest_entry = path / source_entry.name
-        if source_entry.name in present:
-            present.remove(source_entry.name)
-            if dest_entry.is_symlink() and dest_entry.readlink() == source_entry:
-                continue
-            _remove(dest_entry)
-        dest_entry.symlink_to(source_entry)
-    for name in present:
-        dest_entry = path / name
-        _remove(dest_entry)
-
-
-def _install_collections(
-    collections: Iterable[CollectionData], root: Path, *, with_current: bool
-) -> None:
-    for collection in collections:
-        namespace_dir = root / collection.namespace
-        namespace_dir.mkdir(exist_ok=True)
-        path = namespace_dir / collection.name
-        if not collection.current:
-            _install_collection(collection, path)
-        elif with_current:
-            _install_current_collection(collection, path)
-
-
-def _extract_collections_from_extra_deps_file(path: str | os.PathLike) -> list[str]:
-    if not os.path.exists(path):
-        return []
-    try:
-        data = load_yaml_file(path)
-        result = []
-        if data.get("collections"):
-            for index, collection in enumerate(data["collections"]):
-                if isinstance(collection, str):
-                    result.append(collection)
-                    continue
-                if not isinstance(collection, dict):
-                    raise ValueError(
-                        f"Collection entry #{index + 1} must be a string or dictionary"
-                    )
-                if not isinstance(collection.get("name"), str):
-                    raise ValueError(
-                        f"Collection entry #{index + 1} does not have a 'name' field of type string"
-                    )
-                result.append(collection["name"])
-        return result
-    except Exception as exc:
-        raise ValueError(
-            f"Error while loading collection dependency file {path}: {exc}"
-        ) from exc
+    def add_collection(
+        self, *, directory: Path, namespace: str, name: str
+    ) -> CollectionData:
+        """
+        Add a new collection to the cache.
+        """
+        # pylint: disable-next=protected-access
+        return self._owner._add_collection(
+            directory=directory, namespace=namespace, name=name
+        )
 
+    def get_global_cache(self) -> _GlobalCache:
+        """
+        Get the global cache object.
+        """
+        return self._owner._get_global_cache()  # pylint: disable=protected-access
 
-@dataclass
-class SetupResult:
-    """
-    Information on how the collections are set up.
-    """
 
-    # The path of the ansible_collections directory.
-    root: Path
+class _CollectionListSingleton:
+    _lock = threading.Lock()
 
-    # Data on the current collection (as in the repository).
-    current_collection: CollectionData
+    _global_cache_dir: Path | None = None
+    _collection_list: CollectionList | None = None
 
-    # If it was installed, the path of the current collection inside the collection tree below root.
-    current_path: Path | None
+    def setup(self, *, global_cache_dir: Path) -> None:
+        """
+        Setup data.
+        """
+        with self._lock:
+            if (
+                self._global_cache_dir is not None
+                and self._global_cache_dir != global_cache_dir
+            ):
+                raise ValueError(
+                    "Setup mismatch: global cache dir cannot be both"
+                    f" {self._global_cache_dir} and {global_cache_dir}"
+                )
+            self._global_cache_dir = global_cache_dir
 
+    def clear(self) -> None:
+        """
+        Clear collection cache.
+        """
+        with self._lock:
+            self._collection_list = None
 
-def setup_collections(
-    destination: str | os.PathLike,
-    runner: Runner,
-    *,
-    extra_collections: list[str] | None = None,
-    extra_deps_files: list[str | os.PathLike] | None = None,
-    with_current: bool = True,
-) -> SetupResult:
-    """
-    Setup all collections in a tree structure inside the destination directory.
-    """
-    all_collections = get_collection_list(runner)
-    destination_root = Path(destination) / "ansible_collections"
-    destination_root.mkdir(exist_ok=True)
-    current = all_collections.current
-    collections_to_install = {current.full_name: current}
-    if extra_collections:
-        for collection in extra_collections:
-            collection_data = all_collections.find(collection)
-            if collection_data is None:
+    def get_cached(self) -> CollectionList | None:
+        """
+        Return cached list of collections, if present.
+        Do not modify the result!
+        """
+        return self._collection_list
+
+    def get(self, *, runner: Runner) -> CollectionList:
+        """
+        Search for a list of collections. The result is cached.
+        """
+        with self._lock:
+            if self._global_cache_dir is None:
+                raise ValueError("Internal error: global cache dir not setup")
+            result = self._collection_list
+            if result is None:
+                result = CollectionList.collect(
+                    runner=runner,
+                    global_cache=_GlobalCache.create(root=self._global_cache_dir),
+                )
+                self._collection_list = result
+        return result.clone()
+
+    def _get_global_cache(self) -> _GlobalCache:
+        """
+        Returns the global cache dir.
+        """
+        if self._global_cache_dir is None:
+            raise ValueError("Internal error: global cache dir not setup")
+        return _GlobalCache.create(root=self._global_cache_dir)
+
+    def _add_collection(
+        self, *, directory: Path, namespace: str, name: str
+    ) -> CollectionData:
+        """
+        Add collection in directory if the collection list has been cached.
+        """
+        if not self._collection_list:
+            raise ValueError("Internal error: collections not listed")
+        data = load_collection_data_from_disk(directory, namespace=namespace, name=name)
+        self._collection_list._add(data)  # pylint: disable=protected-access
+        return data
+
+    @contextmanager
+    def _update_collection_list(self) -> t.Iterator[_CollectionListUpdater]:
+        with self._lock:
+            if not self._collection_list or self._global_cache_dir is None:
                 raise ValueError(
-                    f"Cannot find collection {collection} required by the noxfile!"
+                    "Internal error: collections not listed or global cache not setup"
                 )
-            collections_to_install[collection_data.full_name] = collection_data
-    if extra_deps_files is not None:
-        for extra_deps_file in extra_deps_files:
-            for collection in _extract_collections_from_extra_deps_file(
-                extra_deps_file
-            ):
-                collection_data = all_collections.find(collection)
-                if collection_data is None:
-                    raise ValueError(
-                        f"Cannot find collection {collection} required in {extra_deps_file}!"
-                    )
-                collections_to_install[collection_data.full_name] = collection_data
-    _add_all_dependencies(collections_to_install, all_collections)
-    _install_collections(
-        collections_to_install.values(), destination_root, with_current=with_current
-    )
-    return SetupResult(
-        root=destination_root,
-        current_collection=current,
-        current_path=(
-            (destination_root / current.namespace / current.name)
-            if with_current
-            else None
-        ),
-    )
+            yield _CollectionListUpdater(
+                owner=self, collection_list=self._collection_list
+            )
 
 
-def _copy_collection(collection: CollectionData, path: Path) -> None:
-    _paths_copy_collection(collection.path, path)
-
-
-def _copy_collection_rsync_hard_links(
-    collection: CollectionData, path: Path, runner: Runner
-) -> None:
-    _, __ = runner(
-        [
-            "rsync",
-            "-av",
-            "--delete",
-            "--exclude",
-            ".nox",
-            "--link-dest",
-            str(collection.path) + "/",
-            "--",
-            str(collection.path) + "/",
-            str(path) + "/",
-        ]
-    )
+_COLLECTION_LIST = _CollectionListSingleton()
 
 
-def setup_current_tree(
-    place: str | os.PathLike, current_collection: CollectionData
-) -> SetupResult:
+@contextmanager
+def _update_collection_list() -> t.Iterator[_CollectionListUpdater]:
+    # pylint: disable-next=protected-access
+    with _COLLECTION_LIST._update_collection_list() as result:
+        yield result
+
+
+def get_collection_list(*, runner: Runner, global_cache_dir: Path) -> CollectionList:
     """
-    Setup a tree structure with the current collection in it.
+    Search for a list of collections. The result is cached.
     """
-
-    path = Path(place)
-    root = path / "ansible_collections"
-    root.mkdir(exist_ok=True)
-    namespace = root / current_collection.namespace
-    namespace.mkdir(exist_ok=True)
-    collection = namespace / current_collection.name
-    _copy_collection(current_collection, collection)
-    # _copy_collection_rsync_hard_links(current_collection, collection, runner)
-    return SetupResult(
-        root=root,
-        current_collection=current_collection,
-        current_path=collection,
-    )
+    _COLLECTION_LIST.setup(global_cache_dir=global_cache_dir)
+    return _COLLECTION_LIST.get(runner=runner)
 
 
 __all__ = [
-    "CollectionData",
     "CollectionList",
-    "SetupResult",
     "get_collection_list",
     "load_collection_data_from_disk",
-    "setup_collections",
-    "setup_current_tree",
 ]