ruyi 0.39.0a20250731__py3-none-any.whl

ruyi/ruyipkg/canonical_dump.py

@@ -0,0 +1,320 @@
+from copy import deepcopy
+import re
+from typing import Final
+
+from tomlkit import comment, document, nl, string, table, ws
+from tomlkit.items import AoT, Array, InlineTable, Table, Trivia
+from tomlkit.toml_document import TOMLDocument
+
+from .pkg_manifest import (
+    BinaryDeclType,
+    BinaryFileDeclType,
+    BlobDeclType,
+    DistfileDeclType,
+    EmulatorDeclType,
+    EmulatorProgramDeclType,
+    FetchRestrictionDeclType,
+    PackageManifest,
+    PackageMetadataDeclType,
+    ProvisionableDeclType,
+    ServiceLevelDeclType,
+    SourceDeclType,
+    ToolchainComponentDeclType,
+    ToolchainDeclType,
+    VendorDeclType,
+)
+from ..utils.toml import (
+    extract_footer_comments,
+    extract_header_comments,
+    inline_table_with_spaces,
+    sorted_table,
+    str_array,
+)
+
+RE_INDENT_FIX: Final = re.compile(r"(?m)^    ([\"'{\[])")
+
+
+# XXX: To workaround https://github.com/python-poetry/tomlkit/issues/290,
+# post-process the output to have all leading 4-space indentation before
+# strings, lists or tables replaced by 2-space ones.
+def _fix_indent(s: str) -> str:
+    return RE_INDENT_FIX.sub(r"  \1", s)
+
+
+def dumps_canonical_package_manifest_toml(
+    pm: PackageManifest,
+) -> str:
+    return _fix_indent(_dump_canonical_package_manifest_toml(pm).as_string())
+
+
+def _dump_canonical_package_manifest_toml(
+    pm: PackageManifest,
+) -> TOMLDocument:
+    x = pm.to_raw()
+    doc = pm.raw_doc
+
+    y = document()
+
+    if doc is not None:
+        if header_comments := extract_header_comments(doc):
+            last_is_ws = False
+            for c in header_comments:
+                if c.startswith("#"):
+                    last_is_ws = False
+                    y.add(comment(c[1:].strip()))
+                else:
+                    last_is_ws = True
+                    y.add(ws(c))
+
+            if not last_is_ws:
+                y.add(nl())
+
+    y.add("format", string(x["format"]))
+
+    dump_metadata_decl_into(y, x["metadata"])
+    dump_distfile_decls_into(y, x["distfiles"])
+    maybe_dump_binary_decls_into(y, x.get("binary"))
+    maybe_dump_blob_decl_into(y, x.get("blob"))
+    maybe_dump_emulator_decl_into(y, x.get("emulator"))
+    maybe_dump_provisionable_decl_into(y, x.get("provisionable"))
+    maybe_dump_source_decl_into(y, x.get("source"))
+    maybe_dump_toolchain_decl_into(y, x.get("toolchain"))
+
+    if doc is not None:
+        if footer_comments := extract_footer_comments(doc):
+            if footer_comments[0].startswith("#"):
+                y.add(nl())
+
+            for c in footer_comments:
+                if c.startswith("#"):
+                    y.add(comment(c[1:].strip()))
+                else:
+                    y.add(ws(c))
+
+    return y
+
+
+def dump_service_level_entry(x: ServiceLevelDeclType) -> Table:
+    y = table()
+    y.add("level", x["level"])
+    if msgid := x.get("msgid"):
+        y.add("msgid", string(msgid))
+    if params := x.get("params"):
+        y.add("params", sorted_table(params))
+    return y
+
+
+def dump_service_level_decls(x: list[ServiceLevelDeclType]) -> AoT:
+    return AoT([dump_service_level_entry(i) for i in x])
+
+
+def dump_metadata_decl(x: PackageMetadataDeclType) -> Table:
+    y = table()
+    y.add("desc", string(x["desc"]))
+    y.add("vendor", dump_vendor_decl(x["vendor"]))
+    if "slug" in x:
+        y.add("slug", string(x["slug"]))
+    if uv := x.get("upstream_version"):
+        y.add("upstream_version", string(uv))
+    if sl := x.get("service_level"):
+        y.add(nl())
+        y.add("service_level", dump_service_level_decls(sl))
+    return y
+
+
+def dump_metadata_decl_into(doc: TOMLDocument, x: PackageMetadataDeclType) -> None:
+    doc.add(nl())
+    doc.add("metadata", dump_metadata_decl(x))
+
+
+def dump_vendor_decl(x: VendorDeclType) -> InlineTable:
+    y = inline_table_with_spaces()
+    with y:
+        y.add("name", string(x["name"]))
+        y.add("eula", string(x["eula"] if x["eula"] is not None else ""))
+    return y
+
+
+def dump_distfile_decls(x: list[DistfileDeclType]) -> AoT:
+    return AoT([dump_distfile_entry(i) for i in x])
+
+
+def dump_distfile_decls_into(doc: TOMLDocument, x: list[DistfileDeclType]) -> None:
+    doc.add(nl())
+    doc.add("distfiles", dump_distfile_decls(x))
+
+
+def dump_distfile_entry(x: DistfileDeclType) -> Table:
+    y = table()
+    y.add("name", x["name"])
+    if v := x.get("unpack"):
+        y.add("unpack", string(v))
+    y.add("size", x["size"])
+    if s := x.get("strip_components"):
+        if s != 1:
+            y.add("strip_components", s)
+    if p := x.get("prefixes_to_unpack"):
+        y.add("prefixes_to_unpack", str_array(p, multiline=len(p) > 1))
+    if "urls" in x:
+        # XXX: https://github.com/python-poetry/tomlkit/issues/290 prevents us
+        # from using 2-space indentation for the array items for now.
+        y.add("urls", str_array([str(i) for i in x["urls"]], multiline=True))
+    if r := x.get("restrict"):
+        # If `restrict` is a string, convert it to a list, fixing a common
+        # oversight in package manifests.
+        if isinstance(r, str):
+            r = [r]
+        y.add("restrict", [str(i) for i in r])
+    if f := x.get("fetch_restriction"):
+        y.add("fetch_restriction", dump_fetch_restriction(f))
+    y.add("checksums", sorted_table(x["checksums"]))
+    return y
+
+
+def dump_fetch_restriction(x: FetchRestrictionDeclType) -> Table:
+    y = table()
+    y.add("msgid", x["msgid"])
+    if "params" in x:
+        y.add("params", sorted_table(x["params"]))
+    return y
+
+
+def dump_blob_decl(x: BlobDeclType) -> Table:
+    y = table()
+    y.add("distfiles", str_array(x["distfiles"], multiline=True))
+    return y
+
+
+def maybe_dump_blob_decl_into(doc: TOMLDocument, x: BlobDeclType | None) -> None:
+    if x is None:
+        return
+    doc.add(nl())
+    doc.add("blob", dump_blob_decl(x))
+
+
+def dump_provisionable_decl(x: ProvisionableDeclType) -> Table:
+    y = table()
+    y.add("strategy", x["strategy"])
+    y.add(
+        "partition_map",
+        sorted_table({str(k): v for k, v in x["partition_map"].items()}),
+    )
+    return y
+
+
+def maybe_dump_provisionable_decl_into(
+    doc: TOMLDocument,
+    x: ProvisionableDeclType | None,
+) -> None:
+    if x is None:
+        return
+    doc.add(nl())
+    doc.add("provisionable", dump_provisionable_decl(x))
+
+
+def dump_binary_decl(x: BinaryFileDeclType, last: bool) -> Table:
+    y = table()
+    y.add("host", string(x["host"]))
+    multiline_distfiles = len(x["distfiles"]) > 1
+    y.add("distfiles", str_array(x["distfiles"], multiline=multiline_distfiles))
+    if cmds := x.get("commands", {}):
+        y.add("commands", sorted_table(cmds))
+        if not last:
+            y.add(nl())
+    return y
+
+
+def dump_binary_decls(x: list[BinaryFileDeclType]) -> AoT:
+    return AoT([dump_binary_decl(elem, i == len(x) - 1) for i, elem in enumerate(x)])
+
+
+def maybe_dump_binary_decls_into(doc: TOMLDocument, x: BinaryDeclType | None) -> None:
+    if x is None:
+        return
+    doc.add("binary", dump_binary_decls(x))
+
+
+def dump_emulator_program_decl(x: EmulatorProgramDeclType) -> Table:
+    y = table()
+    y.add("path", string(x["path"]))
+    y.add("flavor", string(x["flavor"]))
+    y.add("supported_arches", str_array(x["supported_arches"]))
+    if "binfmt_misc" in x:
+        y.add("binfmt_misc", string(x["binfmt_misc"]))
+    return y
+
+
+def dump_emulator_decl(x: EmulatorDeclType) -> Table:
+    y = table()
+    # Prefer `quirks` to `flavors`
+    quirks = x.get("quirks")
+    if quirks is None:
+        quirks = x.get("flavors", [])
+    y.add("quirks", str_array(quirks))
+    y.add("programs", AoT([dump_emulator_program_decl(i) for i in x["programs"]]))
+    return y
+
+
+def maybe_dump_emulator_decl_into(
+    doc: TOMLDocument, x: EmulatorDeclType | None
+) -> None:
+    if x is None:
+        return
+    doc.add(nl())
+    doc.add("emulator", dump_emulator_decl(x))
+
+
+def dump_source_decl(x: SourceDeclType) -> Table:
+    y = table()
+    multiline_distfiles = len(x["distfiles"]) > 1
+    y.add("distfiles", str_array(x["distfiles"], multiline=multiline_distfiles))
+    return y
+
+
+def maybe_dump_source_decl_into(doc: TOMLDocument, x: SourceDeclType | None) -> None:
+    if x is None:
+        return
+    doc.add(nl())
+    doc.add("source", dump_source_decl(x))
+
+
+def dump_toolchain_component_decl(x: ToolchainComponentDeclType) -> InlineTable:
+    y = inline_table_with_spaces()
+    with y:
+        y.add("name", string(x["name"]))
+        y.add("version", string(x["version"]))
+    return y
+
+
+def dump_toolchain_component_decls(x: list[ToolchainComponentDeclType]) -> Array:
+    sorted_x = deepcopy(x)
+    sorted_x.sort(key=lambda i: i["name"])
+    return Array(
+        [dump_toolchain_component_decl(i) for i in sorted_x],
+        Trivia(),
+        multiline=True,
+    )
+
+
+def dump_toolchain_decl(x: ToolchainDeclType) -> Table:
+    y = table()
+    y.add("target", string(x["target"]))
+    # Prefer `quirks` to `flavors`
+    quirks = x.get("quirks")
+    if quirks is None:
+        quirks = x.get("flavors", [])
+    y.add("quirks", str_array(quirks))
+    y.add("components", dump_toolchain_component_decls(x["components"]))
+    if "included_sysroot" in x:
+        y.add("included_sysroot", x["included_sysroot"])
+    return y
+
+
+def maybe_dump_toolchain_decl_into(
+    doc: TOMLDocument,
+    x: ToolchainDeclType | None,
+) -> None:
+    if x is None:
+        return
+    doc.add(nl())
+    doc.add("toolchain", dump_toolchain_decl(x))

ruyi/ruyipkg/checksum.py

@@ -0,0 +1,39 @@
+import hashlib
+from typing import BinaryIO, Final, Iterable
+
+SUPPORTED_CHECKSUM_KINDS: Final = {"sha256", "sha512"}
+
+
+def get_hash_instance(kind: str) -> "hashlib._Hash":
+    if kind not in SUPPORTED_CHECKSUM_KINDS:
+        raise ValueError(f"checksum algorithm {kind} not supported")
+    return hashlib.new(kind)
+
+
+class Checksummer:
+    def __init__(self, file: BinaryIO, checksums: dict[str, str]) -> None:
+        self.file = file
+        self.checksums = checksums
+
+    def check(self) -> None:
+        computed_csums = self.compute()
+        for kind, expected_csum in self.checksums.items():
+            if computed_csums[kind] != expected_csum:
+                raise ValueError(
+                    f"wrong {kind} checksum: want {expected_csum}, got {computed_csums[kind]}"
+                )
+
+    def compute(
+        self,
+        kinds: Iterable[str] | None = None,
+        chunksize: int = 4096,
+    ) -> dict[str, str]:
+        if kinds is None:
+            kinds = self.checksums.keys()
+
+        checksummers = {kind: get_hash_instance(kind) for kind in kinds}
+        while chunk := self.file.read(chunksize):
+            for h in checksummers.values():
+                h.update(chunk)
+
+        return {kind: h.hexdigest() for kind, h in checksummers.items()}

ruyi/ruyipkg/cli_completion.py

@@ -0,0 +1,42 @@
+from typing import Any, Callable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..cli.completer import DynamicCompleter
+    from ..config import GlobalConfig
+
+
+def package_completer_builder(
+    cfg: "GlobalConfig",
+    filters: list[Callable[[str], bool]] | None = None,
+) -> "DynamicCompleter":
+    # Lazy import to avoid circular dependency
+    from ..ruyipkg.augmented_pkg import (
+        AugmentedPkg,
+    )  # pylint: disable=import-outside-toplevel
+    from ..ruyipkg.list_filter import (
+        ListFilter,
+    )  # pylint: disable=import-outside-toplevel
+
+    all_pkgs = list(
+        AugmentedPkg.yield_from_repo(
+            cfg,
+            cfg.repo,
+            ListFilter(),
+            ensure_repo=False,
+        )
+    )
+    if filters is not None:
+        all_pkgs = [
+            pkg
+            for pkg in all_pkgs
+            if pkg.name is not None and all(f(pkg.name) for f in filters)
+        ]
+
+    def f(prefix: str, parsed_args: object, **kwargs: Any) -> list[str]:
+        return [
+            pkg.name
+            for pkg in all_pkgs
+            if pkg.name is not None and pkg.name.startswith(prefix)
+        ]
+
+    return f

ruyi/ruyipkg/distfile.py

@@ -0,0 +1,208 @@
+from functools import cached_property
+import os
+from typing import Final
+
+from ..log import RuyiLogger
+from .checksum import Checksummer
+from .fetch import BaseFetcher
+from .pkg_manifest import DistfileDecl
+from .repo import MetadataRepo
+from .unpack import do_unpack, do_unpack_or_symlink
+from .unpack_method import UnpackMethod
+
+
+# https://github.com/ruyisdk/ruyi/issues/46
+HELP_ERROR_FETCHING: Final = """
+Downloads can fail for a multitude of reasons, most of which should not and
+cannot be handled by [yellow]Ruyi[/]. For your convenience though, please check if any
+of the following common failure modes apply to you, and take actions
+accordingly if one of them turns out to be the case:
+
+* Basic connectivity problems
+    - is [yellow]the gateway[/] reachable?
+    - is [yellow]common websites[/] reachable?
+    - is there any [yellow]DNS pollution[/]?
+* Organizational and/or ISP restrictions
+    - is there a [yellow]firewall[/] preventing Ruyi traffic?
+    - is your [yellow]ISP blocking access[/] to the source website?
+* Volatile upstream
+    - is the recorded [yellow]link dead[/]? (Please raise a Ruyi issue for a fix!)
+"""
+
+
+class Distfile:
+    def __init__(
+        self,
+        decl: DistfileDecl,
+        mr: MetadataRepo,
+    ) -> None:
+        self._decl = decl
+        self._mr = mr
+
+    @cached_property
+    def dest(self) -> str:
+        destdir = self._mr.global_config.ensure_distfiles_dir()
+        return os.path.join(destdir, self._decl.name)
+
+    @property
+    def size(self) -> int:
+        return self._decl.size
+
+    @property
+    def csums(self) -> dict[str, str]:
+        return self._decl.checksums
+
+    @property
+    def prefixes_to_unpack(self) -> list[str] | None:
+        return self._decl.prefixes_to_unpack
+
+    @property
+    def strip_components(self) -> int:
+        return self._decl.strip_components
+
+    @property
+    def unpack_method(self) -> UnpackMethod:
+        return self._decl.unpack_method
+
+    @property
+    def is_fetch_restricted(self) -> bool:
+        return self._decl.is_restricted("fetch")
+
+    @cached_property
+    def urls(self) -> list[str]:
+        return self._mr.get_distfile_urls(self._decl)
+
+    def render_fetch_instructions(self, logger: RuyiLogger, lang_code: str) -> str:
+        fr = self._decl.fetch_restriction
+        if fr is None:
+            return ""
+
+        params = {
+            "dest_path": self.dest,
+        }
+        if "params" in fr:
+            for k in params.keys():
+                # Don't allow package-defined params to override preset params,
+                # to reduce surprises for packagers.
+                if k in fr["params"]:
+                    logger.F(
+                        f"malformed package fetch instructions: the param named '{k}' is reserved and cannot be overridden by packages"
+                    )
+                    raise RuntimeError("malformed package fetch instructions")
+
+            params.update(fr["params"])
+
+        return self._mr.messages.render_message(fr["msgid"], lang_code, params)
+
+    def is_downloaded(self) -> bool:
+        """Check if the distfile has been downloaded. A return value of True
+        does NOT guarantee integrity."""
+
+        try:
+            st = os.stat(self.dest)
+            return st.st_size == self.size
+        except FileNotFoundError:
+            return False
+
+    def ensure(self, logger: RuyiLogger) -> None:
+        logger.D(f"checking {self.dest}")
+        try:
+            st = os.stat(self.dest)
+        except FileNotFoundError:
+            logger.D(f"file {self.dest} not existent")
+            return self.fetch_and_ensure_integrity(logger)
+
+        if st.st_size < self.size:
+            # assume incomplete transmission, try to resume
+            logger.D(
+                f"file {self.dest} appears incomplete: size {st.st_size} < {self.size}; resuming"
+            )
+            return self.fetch_and_ensure_integrity(logger, resume=True)
+        elif st.st_size == self.size:
+            if self.ensure_integrity_or_rm(logger):
+                logger.D(f"file {self.dest} passed checks")
+                return
+
+            # the file is already gone, re-fetch
+            logger.D(f"re-fetching {self.dest}")
+            return self.fetch_and_ensure_integrity(logger)
+
+        logger.W(
+            f"file {self.dest} is corrupt: size too big ({st.st_size} > {self.size}); deleting"
+        )
+        os.remove(self.dest)
+        return self.fetch_and_ensure_integrity(logger)
+
+    def ensure_integrity_or_rm(self, logger: RuyiLogger) -> bool:
+        try:
+            with open(self.dest, "rb") as fp:
+                cs = Checksummer(fp, self.csums)
+                cs.check()
+                return True
+        except ValueError as e:
+            logger.W(f"file {self.dest} is corrupt: {e}; deleting")
+            os.remove(self.dest)
+            return False
+
+    def fetch_and_ensure_integrity(
+        self,
+        logger: RuyiLogger,
+        *,
+        resume: bool = False,
+    ) -> None:
+        if self.is_fetch_restricted:
+            # the file must be re-fetched if we arrive here, but we cannot,
+            # because of the fetch restriction.
+            #
+            # notify the user and die
+            # TODO: allow rendering instructions for all missing fetch-restricted
+            # files at once
+            logger.F(
+                f"the file [yellow]'{self.dest}'[/] cannot be automatically fetched"
+            )
+            logger.I("instructions on fetching this file:")
+            logger.I(
+                self.render_fetch_instructions(logger, self._mr.global_config.lang_code)
+            )
+            raise SystemExit(1)
+
+        try:
+            return self._fetch_and_ensure_integrity(logger, resume=resume)
+        except RuntimeError as e:
+            logger.F(f"{e}")
+            logger.stdout(HELP_ERROR_FETCHING)
+            raise SystemExit(1)
+
+    def _fetch_and_ensure_integrity(
+        self,
+        logger: RuyiLogger,
+        *,
+        resume: bool = False,
+    ) -> None:
+        fetcher = BaseFetcher.new(logger, self.urls, self.dest)
+        fetcher.fetch(resume=resume)
+
+        if not self.ensure_integrity_or_rm(logger):
+            raise RuntimeError(
+                f"failed to fetch distfile: {self.dest} failed integrity checks"
+            )
+
+    def unpack(self, root: str | None, logger: RuyiLogger) -> None:
+        return do_unpack(
+            logger,
+            self.dest,
+            root,
+            self.strip_components,
+            self.unpack_method,
+            prefixes_to_unpack=self.prefixes_to_unpack,
+        )
+
+    def unpack_or_symlink(self, root: str | None, logger: RuyiLogger) -> None:
+        return do_unpack_or_symlink(
+            logger,
+            self.dest,
+            root,
+            self.strip_components,
+            self.unpack_method,
+            prefixes_to_unpack=self.prefixes_to_unpack,
+        )