ruyi 0.39.0__py3-none-any.whl

ruyi/ruyipkg/state.py

@@ -0,0 +1,345 @@
+import datetime
+import json
+import os
+import pathlib
+from typing import Any, Iterable, Iterator, TypedDict, TYPE_CHECKING
+from dataclasses import dataclass
+
+from .pkg_manifest import BoundPackageManifest
+from .protocols import ProvidesPackageManifests
+
+if TYPE_CHECKING:
+    # for avoiding heavy import
+    from .repo import MetadataRepo
+
+
+class PackageInstallationRecord(TypedDict):
+    """Record of a package installation."""
+
+    repo_id: str
+    category: str
+    name: str
+    version: str
+    host: str  # For binary packages, empty for blobs
+    install_path: str
+    install_time: str  # ISO format datetime
+
+
+@dataclass
+class PackageInstallationInfo:
+    """Information about an installed package."""
+
+    repo_id: str
+    category: str
+    name: str
+    version: str
+    host: str  # For binary packages, empty for blobs
+    install_path: str
+    install_time: datetime.datetime
+
+    def to_record(self) -> PackageInstallationRecord:
+        """Convert to a record for JSON serialization."""
+        return PackageInstallationRecord(
+            repo_id=self.repo_id,
+            category=self.category,
+            name=self.name,
+            version=self.version,
+            host=self.host,
+            install_path=self.install_path,
+            install_time=self.install_time.isoformat(),
+        )
+
+    @classmethod
+    def from_record(
+        cls,
+        record: PackageInstallationRecord,
+    ) -> "PackageInstallationInfo":
+        """Create from a record."""
+        return cls(
+            repo_id=record["repo_id"],
+            category=record["category"],
+            name=record["name"],
+            version=record["version"],
+            host=record["host"],
+            install_path=record["install_path"],
+            install_time=datetime.datetime.fromisoformat(record["install_time"]),
+        )
+
+
+class RuyipkgGlobalStateStore:
+    def __init__(self, root: os.PathLike[Any]) -> None:
+        self.root = pathlib.Path(root)
+        self._installs_file = self.root / "installs.json"
+        self._installs_cache: dict[str, PackageInstallationInfo] | None = None
+
+    def ensure_state_dir(self) -> None:
+        """Ensure the state directory exists."""
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def _load_installs(self) -> dict[str, PackageInstallationInfo]:
+        """Load installation records from disk."""
+        if self._installs_cache is not None:
+            return self._installs_cache
+
+        self.ensure_state_dir()
+
+        if not self._installs_file.exists():
+            self._installs_cache = {}
+            return self._installs_cache
+
+        try:
+            with open(self._installs_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+            installs = {}
+            for key, record in data.items():
+                installs[key] = PackageInstallationInfo.from_record(record)
+
+            self._installs_cache = installs
+            return self._installs_cache
+        except (json.JSONDecodeError, KeyError, ValueError):
+            # If file is corrupted, start fresh
+            self._installs_cache = {}
+            return self._installs_cache
+
+    def _save_installs(self) -> None:
+        """Save installation records to disk."""
+        if self._installs_cache is None:
+            return
+
+        self.ensure_state_dir()
+
+        data = {}
+        for key, info in self._installs_cache.items():
+            data[key] = info.to_record()
+
+        # Write atomically by writing to temp file then renaming
+        temp_file = self._installs_file.with_suffix(".tmp")
+        try:
+            with open(temp_file, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=2, ensure_ascii=False)
+            temp_file.replace(self._installs_file)
+        except Exception:
+            # Clean up temp file if something went wrong
+            if temp_file.exists():
+                temp_file.unlink()
+            raise
+
+    def _get_installation_key(
+        self,
+        repo_id: str,
+        category: str,
+        name: str,
+        version: str,
+        host: str = "",
+    ) -> str:
+        """Get the key used to store installation info."""
+        if host:
+            # Use a format that includes host for binary packages
+            return f"{repo_id}:{category}/{name} {version} host={host}"
+        return f"{repo_id}:{category}/{name} {version}"
+
+    def record_installation(
+        self,
+        repo_id: str,
+        category: str,
+        name: str,
+        version: str,
+        host: str,
+        install_path: str,
+    ) -> None:
+        """Record a successful package installation."""
+        installs = self._load_installs()
+
+        key = self._get_installation_key(repo_id, category, name, version, host)
+        info = PackageInstallationInfo(
+            repo_id=repo_id,
+            category=category,
+            name=name,
+            version=version,
+            host=host,
+            install_path=install_path,
+            install_time=datetime.datetime.now(),
+        )
+
+        installs[key] = info
+        self._save_installs()
+
+    def remove_installation(
+        self,
+        repo_id: str,
+        category: str,
+        name: str,
+        version: str,
+        host: str = "",
+    ) -> bool:
+        """Remove an installation record."""
+        installs = self._load_installs()
+        key = self._get_installation_key(repo_id, category, name, version, host)
+
+        if key in installs:
+            del installs[key]
+            self._save_installs()
+            return True
+        return False
+
+    def get_installation(
+        self,
+        repo_id: str,
+        category: str,
+        name: str,
+        version: str,
+        host: str = "",
+    ) -> PackageInstallationInfo | None:
+        """Get information about a specific installation."""
+        installs = self._load_installs()
+        key = self._get_installation_key(repo_id, category, name, version, host)
+        return installs.get(key)
+
+    def is_package_installed(
+        self,
+        repo_id: str,
+        category: str,
+        name: str,
+        version: str,
+        host: str = "",
+    ) -> bool:
+        """Check if a package is installed."""
+        return self.get_installation(repo_id, category, name, version, host) is not None
+
+    def list_installed_packages(self) -> list[PackageInstallationInfo]:
+        """List all installed packages."""
+        installs = self._load_installs()
+        return list(installs.values())
+
+
+class BoundInstallationStateStore(ProvidesPackageManifests):
+    def __init__(self, rgs: RuyipkgGlobalStateStore, mr: "MetadataRepo") -> None:
+        self._rgs = rgs
+        self._mr = mr
+
+    def _get_installed_manifest(
+        self,
+        info: PackageInstallationInfo,
+    ) -> BoundPackageManifest | None:
+        """Get the bound manifest for an installed package, or None if not found in repo."""
+
+        return self._mr.get_pkg(info.name, info.category, info.version)
+
+    def iter_pkg_manifests(self) -> Iterable[BoundPackageManifest]:
+        """Iterate over all installed package manifests."""
+
+        installed_pkgs = self._rgs.list_installed_packages()
+        for info in installed_pkgs:
+            if m := self._get_installed_manifest(info):
+                yield m
+
+    def iter_pkgs(
+        self,
+    ) -> Iterable[tuple[str, str, dict[str, BoundPackageManifest]]]:
+        """Iterate over installed packages grouped by category and name."""
+
+        installed_pkgs = self._rgs.list_installed_packages()
+
+        # Group by category and name
+        result: dict[str, dict[str, dict[str, BoundPackageManifest]]] = {}
+        for info in installed_pkgs:
+            if m := self._get_installed_manifest(info):
+                if info.category not in result:
+                    result[info.category] = {}
+                if info.name not in result[info.category]:
+                    result[info.category][info.name] = {}
+                result[info.category][info.name][info.version] = m
+
+        for category, cat_pkgs in result.items():
+            for pkg_name, pkg_vers in cat_pkgs.items():
+                yield (category, pkg_name, pkg_vers)
+
+    def iter_pkg_vers(
+        self,
+        name: str,
+        category: str | None = None,
+    ) -> Iterable[BoundPackageManifest]:
+        """Iterate over installed versions of a specific package."""
+
+        installed_pkgs = self._rgs.list_installed_packages()
+        for info in installed_pkgs:
+            if info.name == name and (category is None or info.category == category):
+                if m := self._get_installed_manifest(info):
+                    yield m
+
+    def get_pkg(
+        self,
+        name: str,
+        category: str,
+        ver: str,
+    ) -> BoundPackageManifest | None:
+        """Returns the package manifest by exact match, or None if not found."""
+        installed_pkgs = self._rgs.list_installed_packages()
+        for info in installed_pkgs:
+            if info.name == name and info.category == category and info.version == ver:
+                if m := self._get_installed_manifest(info):
+                    return m
+                # Package is installed but not found in current repo
+                break
+
+        return None
+
+    def get_pkg_latest_ver(
+        self,
+        name: str,
+        category: str | None = None,
+        include_prerelease_vers: bool = False,
+    ) -> BoundPackageManifest:
+        """Get the latest installed version of a package."""
+
+        from .pkg_manifest import is_prerelease
+
+        installed_vers = list(self.iter_pkg_vers(name, category))
+        if not installed_vers:
+            raise KeyError(f"No installed versions found for package '{name}'")
+
+        if not include_prerelease_vers:
+            installed_vers = [
+                pm for pm in installed_vers if not is_prerelease(pm.semver)
+            ]
+            if not installed_vers:
+                raise KeyError(
+                    f"No non-prerelease installed versions found for package '{name}'"
+                )
+
+        # Find the latest version
+        latest = max(installed_vers, key=lambda pm: pm.semver)
+        return latest
+
+    # To be removed later along with slug support
+    def get_pkg_by_slug(self, slug: str) -> BoundPackageManifest | None:
+        """Get an installed package by its slug."""
+
+        installed_pkgs = self._rgs.list_installed_packages()
+        for info in installed_pkgs:
+            if m := self._get_installed_manifest(info):
+                if m.slug == slug:
+                    return m
+        return None
+
+    # Useful helpers
+
+    def iter_upgradable_pkgs(
+        self,
+        include_prereleases: bool = False,
+    ) -> Iterator[tuple[BoundPackageManifest, str]]:
+        for installed_pm in self.iter_pkg_manifests():
+            latest_pm: BoundPackageManifest
+            try:
+                latest_pm = self._mr.get_pkg_latest_ver(
+                    installed_pm.name,
+                    installed_pm.category,
+                    include_prereleases,
+                )
+            except KeyError:
+                # package not found in the repo, skip it
+                continue
+
+            if latest_pm.semver > installed_pm.semver:
+                yield (installed_pm, str(latest_pm.semver))

ruyi/ruyipkg/unpack.py

@@ -0,0 +1,369 @@
+import mmap
+import os
+import shutil
+import subprocess
+from typing import BinaryIO, NoReturn, Protocol
+
+from ..log import RuyiLogger
+from ..utils import ar, prereqs
+from .unpack_method import (
+    UnpackMethod,
+    UnrecognizedPackFormatError,
+    determine_unpack_method,
+)
+
+
+class SupportsRead(Protocol):
+    def read(self, n: int = -1, /) -> bytes: ...
+
+
+def do_unpack(
+    logger: RuyiLogger,
+    filename: str,
+    dest: str | None,
+    strip_components: int,
+    unpack_method: UnpackMethod,
+    stream: BinaryIO | SupportsRead | None = None,
+    prefixes_to_unpack: list[str] | None = None,
+) -> None:
+    match unpack_method:
+        case UnpackMethod.AUTO:
+            raise ValueError("the auto unpack method must be resolved prior to use")
+        case UnpackMethod.RAW:
+            return _do_copy_raw(filename, dest)
+        case (
+            UnpackMethod.TAR_AUTO
+            | UnpackMethod.TAR
+            | UnpackMethod.TAR_BZ2
+            | UnpackMethod.TAR_GZ
+            | UnpackMethod.TAR_LZ4
+            | UnpackMethod.TAR_XZ
+            | UnpackMethod.TAR_ZST
+        ):
+            return _do_unpack_tar(
+                logger,
+                filename,
+                dest,
+                strip_components,
+                unpack_method,
+                stream,
+                prefixes_to_unpack,
+            )
+        case UnpackMethod.ZIP:
+            # TODO: handle strip_components somehow; the unzip(1) command currently
+            # does not have such support.
+            return _do_unpack_zip(logger, filename, dest)
+        case UnpackMethod.DEB:
+            return _do_unpack_deb(logger, filename, dest)
+        case UnpackMethod.GZ:
+            # bare gzip file
+            return _do_unpack_bare_gz(logger, filename, dest)
+        case UnpackMethod.BZ2:
+            # bare bzip2 file
+            return _do_unpack_bare_bzip2(logger, filename, dest)
+        case UnpackMethod.LZ4:
+            # bare lz4 file
+            return _do_unpack_bare_lz4(logger, filename, dest)
+        case UnpackMethod.XZ:
+            # bare xz file
+            return _do_unpack_bare_xz(logger, filename, dest)
+        case UnpackMethod.ZST:
+            # bare zstd file
+            return _do_unpack_bare_zstd(logger, filename, dest)
+        case _:
+            raise UnrecognizedPackFormatError(filename)
+
+
+def do_unpack_or_symlink(
+    logger: RuyiLogger,
+    filename: str,
+    dest: str | None,
+    strip_components: int,
+    unpack_method: UnpackMethod,
+    stream: BinaryIO | SupportsRead | None = None,
+    prefixes_to_unpack: list[str] | None = None,
+) -> None:
+    try:
+        return do_unpack(
+            logger,
+            filename,
+            dest,
+            strip_components,
+            unpack_method,
+            stream,
+            prefixes_to_unpack,
+        )
+    except UnrecognizedPackFormatError:
+        # just symlink into destination
+        return do_symlink(filename, dest)
+
+
+def _do_copy_raw(
+    src_path: str,
+    destdir: str | None,
+) -> None:
+    src_filename = os.path.basename(src_path)
+    if destdir is None:
+        # symlink into CWD
+        dest = src_filename
+    else:
+        dest = os.path.join(destdir, src_filename)
+
+    shutil.copy(src_path, dest)
+
+
+def do_symlink(
+    src_path: str,
+    destdir: str | None,
+) -> None:
+    src_filename = os.path.basename(src_path)
+    if destdir is None:
+        # symlink into CWD
+        dest = src_filename
+    else:
+        dest = os.path.join(destdir, src_filename)
+
+    # avoid the hassle and pitfalls around relative paths and symlinks, and
+    # just point to the target using absolute path
+    symlink_target = os.path.abspath(src_path)
+    os.symlink(symlink_target, dest)
+
+
+def _do_unpack_tar(
+    logger: RuyiLogger,
+    filename: str,
+    dest: str | None,
+    strip_components: int,
+    unpack_method: UnpackMethod,
+    stream: SupportsRead | None,
+    prefixes_to_unpack: list[str] | None = None,
+) -> None:
+    argv = ["tar", "-x"]
+
+    match unpack_method:
+        case UnpackMethod.TAR | UnpackMethod.TAR_AUTO:
+            pass
+        case UnpackMethod.TAR_GZ:
+            argv.append("-z")
+        case UnpackMethod.TAR_BZ2:
+            argv.append("-j")
+        case UnpackMethod.TAR_LZ4:
+            argv.append("--use-compress-program=lz4")
+        case UnpackMethod.TAR_XZ:
+            argv.append("-J")
+        case UnpackMethod.TAR_ZST:
+            argv.append("--zstd")
+        case _:
+            raise ValueError(
+                f"do_unpack_tar cannot handle non-tar unpack method {unpack_method}"
+            )
+
+    stdin: int | None = None
+    if stream is not None:
+        filename = "-"
+        stdin = subprocess.PIPE
+
+    argv.extend(("-f", filename, f"--strip-components={strip_components}"))
+    if dest is not None:
+        argv.extend(("-C", dest))
+    if prefixes_to_unpack:
+        if any(p.startswith("-") for p in prefixes_to_unpack):
+            raise ValueError(
+                "prefixes_to_unpack must not contain any item starting with '-'"
+            )
+        argv.extend(prefixes_to_unpack)
+    logger.D(f"about to call tar: argv={argv}")
+    p = subprocess.Popen(argv, cwd=dest, stdin=stdin)
+
+    retcode: int
+    if stream is None:
+        retcode = p.wait()
+    else:
+        # this is only for pleasing the type-checker; it's statically true
+        # because the assignment always happens due to the earlier
+        # "stream is not None" branch.
+        assert p.stdin is not None
+
+        bufsize = 4 * mmap.PAGESIZE
+        while True:
+            buf = stream.read(bufsize)
+            if not buf:
+                break
+            p.stdin.write(buf)
+        p.stdin.close()
+        retcode = p.wait()
+
+    if retcode != 0:
+        raise RuntimeError(f"untar failed: command {' '.join(argv)} returned {retcode}")
+
+
+def _do_unpack_zip(
+    logger: RuyiLogger,
+    filename: str,
+    dest: str | None,
+) -> None:
+    argv = ["unzip", filename]
+    if dest is not None:
+        argv.extend(("-d", dest))
+    logger.D(f"about to call unzip: argv={argv}")
+    retcode = subprocess.call(argv, cwd=dest)
+    if retcode != 0:
+        raise RuntimeError(f"unzip failed: command {' '.join(argv)} returned {retcode}")
+
+
+def _do_unpack_bare_gz(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    # the suffix may not be ".gz" so do this generically
+    dest_filename = os.path.splitext(os.path.basename(filename))[0]
+
+    argv = ["gunzip", "-c", filename]
+    if destdir is not None:
+        os.chdir(destdir)
+
+    logger.D(f"about to call gunzip: argv={argv}")
+    with open(dest_filename, "wb") as out:
+        retcode = subprocess.call(argv, stdout=out)
+        if retcode != 0:
+            raise RuntimeError(
+                f"gunzip failed: command {' '.join(argv)} returned {retcode}"
+            )
+
+
+def _do_unpack_bare_bzip2(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    # the suffix may not be ".bz2" so do this generically
+    dest_filename = os.path.splitext(os.path.basename(filename))[0]
+
+    argv = ["bzip2", "-dc", filename]
+    if destdir is not None:
+        os.chdir(destdir)
+
+    logger.D(f"about to call bzip2: argv={argv}")
+    with open(dest_filename, "wb") as out:
+        retcode = subprocess.call(argv, stdout=out)
+        if retcode != 0:
+            raise RuntimeError(
+                f"bzip2 failed: command {' '.join(argv)} returned {retcode}"
+            )
+
+
+def _do_unpack_bare_lz4(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    # the suffix may not be ".lz4" so do this generically
+    dest_filename = os.path.splitext(os.path.basename(filename))[0]
+
+    argv = ["lz4", "-dk", filename, f"./{dest_filename}"]
+    logger.D(f"about to call lz4: argv={argv}")
+    retcode = subprocess.call(argv, cwd=destdir)
+    if retcode != 0:
+        raise RuntimeError(f"lz4 failed: command {' '.join(argv)} returned {retcode}")
+
+
+def _do_unpack_bare_xz(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    # the suffix may not be ".xz" so do this generically
+    dest_filename = os.path.splitext(os.path.basename(filename))[0]
+
+    argv = ["xz", "-d", "-c", filename]
+    if destdir is not None:
+        os.chdir(destdir)
+
+    logger.D(f"about to call xz: argv={argv}")
+    with open(dest_filename, "wb") as out:
+        retcode = subprocess.call(argv, stdout=out)
+        if retcode != 0:
+            raise RuntimeError(
+                f"xz failed: command {' '.join(argv)} returned {retcode}"
+            )
+
+
+def _do_unpack_bare_zstd(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    # the suffix may not be ".zst" so do this generically
+    dest_filename = os.path.splitext(os.path.basename(filename))[0]
+
+    argv = ["zstd", "-d", filename, "-o", f"./{dest_filename}"]
+    logger.D(f"about to call zstd: argv={argv}")
+    retcode = subprocess.call(argv, cwd=destdir)
+    if retcode != 0:
+        raise RuntimeError(f"zstd failed: command {' '.join(argv)} returned {retcode}")
+
+
+def _do_unpack_deb(
+    logger: RuyiLogger,
+    filename: str,
+    destdir: str | None,
+) -> None:
+    with ar.ArpyArchiveWrapper(filename) as a:
+        for f in a.infolist():
+            name = f.name.decode("utf-8")
+            if name.startswith("data.tar"):
+                inner_unpack_method = determine_unpack_method(name)
+                return _do_unpack_tar(
+                    logger,
+                    name,
+                    destdir,
+                    0,
+                    inner_unpack_method,
+                    a.open(f),
+                )
+
+    raise RuntimeError(f"file '{filename}' does not appear to be a deb")
+
+
+def _get_unpack_cmds_for_method(m: UnpackMethod) -> list[str]:
+    match m:
+        case UnpackMethod.UNKNOWN | UnpackMethod.RAW | UnpackMethod.DEB:
+            return []
+        case UnpackMethod.GZ:
+            return ["gunzip"]
+        case UnpackMethod.BZ2:
+            return ["bzip2"]
+        case UnpackMethod.LZ4:
+            return ["lz4"]
+        case UnpackMethod.XZ:
+            return ["xz"]
+        case UnpackMethod.ZST:
+            return ["zstd"]
+        case UnpackMethod.TAR | UnpackMethod.TAR_AUTO:
+            return ["tar"]
+        case UnpackMethod.TAR_GZ:
+            return ["tar", "gunzip"]
+        case UnpackMethod.TAR_BZ2:
+            return ["tar", "bzip2"]
+        case UnpackMethod.TAR_LZ4:
+            return ["tar", "lz4"]
+        case UnpackMethod.TAR_XZ:
+            return ["tar", "xz"]
+        case UnpackMethod.TAR_ZST:
+            return ["tar", "zstd"]
+        case UnpackMethod.ZIP:
+            return ["unzip"]
+        case UnpackMethod.AUTO:
+            raise ValueError(f"the unpack method {m} must be resolved prior to use")
+
+
+def ensure_unpack_cmd_for_method(
+    logger: RuyiLogger,
+    m: UnpackMethod,
+) -> None | NoReturn:
+    required_cmds = _get_unpack_cmds_for_method(m)
+    if not required_cmds:
+        return None
+
+    return prereqs.ensure_cmds(logger, required_cmds, interactive_retry=True)