ruyi 0.39.0__py3-none-any.whl

ruyi/ruyipkg/entity_provider.py

@@ -0,0 +1,273 @@
+import abc
+import json
+import os
+import pathlib
+import sys
+from typing import Any, Mapping, Sequence
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+from ..log import RuyiLogger
+from ..utils.porcelain import PorcelainEntity, PorcelainEntityType
+
+
+class PorcelainEntityListOutputV1(PorcelainEntity):
+    entity_type: str
+    entity_id: str
+    display_name: str | None
+    data: Mapping[str, Any]
+    related_refs: list[str]
+    reverse_refs: list[str]
+
+
+class EntityError(Exception):
+    """Base exception for entity-related errors."""
+
+    pass
+
+
+class EntityValidationError(EntityError):
+    """Exception raised when an entity fails validation."""
+
+    def __init__(self, entity_type: str, entity_id: str, cause: Exception) -> None:
+        self.entity_type = entity_type
+        self.entity_id = entity_id
+        self.cause = cause
+        message = (
+            f"Entity validation failed for entity '{entity_type}:{entity_id}': {cause}"
+        )
+        super().__init__(message)
+
+
+class BaseEntity:
+    """Base class for all entity types."""
+
+    def __init__(
+        self,
+        entity_type: str,
+        entity_id: str,
+        data: Mapping[str, Any],
+    ) -> None:
+        self._entity_type = entity_type
+        self._id = entity_id
+        self._data = data
+
+        self._reverse_refs: set[str] = set()
+
+    @property
+    def entity_type(self) -> str:
+        """Type of the entity."""
+        return self._entity_type
+
+    @property
+    def id(self) -> str:
+        """ID of the entity."""
+        return self._id
+
+    @property
+    def display_name(self) -> str | None:
+        """Human-readable name of the entity."""
+        result = self._data[self.entity_type].get("display_name", None)
+        if result is None or isinstance(result, str):
+            return result
+        # return None if type is unexpected
+        return None
+
+    @property
+    def unique_among_type_during_traversal(self) -> bool:
+        """Whether the entity should be unique among all entities of the same type
+        during traversal.
+
+        For example, if the entity is ``arch:foo64`` and there is also ``arch:foo32``,
+        with this property set to ``True`` on each, there will be only one
+        ``arch:foo*`` entity in any traversal path involving them, so that a
+        hypothetical traversal starting from a "foo64" device will not return
+        entities only related to the "foo32" architecture.
+        """
+
+        if r := self._data.get("unique_among_type_during_traversal", None):
+            if isinstance(r, bool):
+                return r
+        # return False if type is unexpected
+        return False
+
+    @property
+    def data(self) -> Any:
+        """Raw data of the entity."""
+        return self._data[self.entity_type]
+
+    @property
+    def related_refs(self) -> list[str]:
+        """Get the list of related entity references."""
+        if r := self._data.get("related"):
+            if isinstance(r, list):
+                return r
+        # return empty list if that is the case, or if the type is unexpected
+        return []
+
+    @property
+    def reverse_refs(self) -> list[str]:
+        """Get the list of reverse-related entity references."""
+        return list(self._reverse_refs)
+
+    def _add_reverse_ref(self, ref: str) -> None:
+        self._reverse_refs.add(ref)
+
+    def to_porcelain(self) -> PorcelainEntityListOutputV1:
+        """Convert this entity to porcelain output format."""
+
+        return {
+            "ty": PorcelainEntityType.EntityListOutputV1,
+            "entity_type": self.entity_type,
+            "entity_id": self.id,
+            "display_name": self.display_name,
+            "data": self._data,
+            "related_refs": self.related_refs,
+            "reverse_refs": self.reverse_refs,
+        }
+
+    def __str__(self) -> str:
+        return f"{self.entity_type}:{self.id}"
+
+    def __hash__(self) -> int:
+        return hash((self.entity_type, self.id))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, BaseEntity):
+            return NotImplemented
+        return self.entity_type == other.entity_type and self.id == other.id
+
+
+class BaseEntityProvider(abc.ABC):
+    """Abstract base class for entity data providers.
+
+    Entity providers are responsible for discovering and loading entity schemas and data.
+    """
+
+    @abc.abstractmethod
+    def discover_schemas(self) -> dict[str, object]:
+        """Discover available entity schemas.
+
+        Returns:
+            A dictionary mapping entity types to their schema objects
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load_entities(
+        self,
+        entity_types: Sequence[str],
+    ) -> Mapping[str, Mapping[str, Mapping[str, Any]]]:
+        """Load entities of the given types.
+
+        Args:
+            entity_types: Sequence of entity types to load
+
+        Returns:
+            A nested dictionary mapping entity types to entity IDs to raw entity data
+        """
+        raise NotImplementedError
+
+
+class FSEntityProvider(BaseEntityProvider):
+    """Entity provider that loads entity data from the filesystem.
+
+    This provider reads schemas from the ``_schemas`` directory and entity data from
+    subdirectories organized by entity type.
+    """
+
+    def __init__(self, logger: RuyiLogger, entities_root: os.PathLike[Any]) -> None:
+        """Initialize the filesystem-based entity provider.
+
+        Args:
+            logger: Logger instance to use.
+            entities_root: Path to the root directory containing entity data.
+                           The ``_schemas`` directory should be a subdirectory of this path.
+        """
+
+        self._logger = logger
+        self._entities_root = pathlib.Path(entities_root)
+        self._schemas_root = self._entities_root / "_schemas"
+
+    def discover_schemas(self) -> dict[str, object]:
+        """Discover entity schemas from the filesystem.
+
+        Returns:
+            A dictionary mapping entity types to their schema objects
+        """
+        schemas: dict[str, object] = {}
+
+        if not os.path.isdir(self._schemas_root):
+            self._logger.D(f"entity schemas directory not found: {self._schemas_root}")
+            return schemas
+
+        try:
+            schema_files = list(self._schemas_root.glob("*.jsonschema"))
+        except IOError as e:
+            self._logger.W(
+                f"failed to access entity schemas directory {self._schemas_root}: {e}"
+            )
+            return schemas
+
+        for p in schema_files:
+            # Extract entity type from schema filename (remove .jsonschema extension)
+            entity_type = p.name[:-11]  # 11 is the length of ".jsonschema"
+
+            try:
+                with open(p, "r", encoding="utf-8") as f:
+                    schema = json.load(f)
+            except (IOError, json.JSONDecodeError) as e:
+                self._logger.D(
+                    f"failed to load schema for entity type '{entity_type}': {e}"
+                )
+                continue
+
+            # Cache the schema
+            schemas[entity_type] = schema
+
+        self._logger.D(f"discovered entity types from schemas: {list(schemas.keys())}")
+        return schemas
+
+    def load_entities(
+        self,
+        entity_types: Sequence[str],
+    ) -> Mapping[str, Mapping[str, Mapping[str, Any]]]:
+        """Load entity data from the filesystem.
+
+        Args:
+            entity_types: Set of entity types to load
+
+        Returns:
+            A nested dictionary mapping entity types to entity IDs to raw entity data
+        """
+        entities: dict[str, dict[str, dict[str, Any]]] = {
+            entity_type: {} for entity_type in entity_types
+        }
+
+        for entity_type in entity_types:
+            type_dir = self._entities_root / entity_type
+
+            if not type_dir.exists():
+                self._logger.D(f"entity type directory does not exist: {type_dir}")
+                continue
+
+            for file_path in type_dir.glob("*.toml"):
+                try:
+                    with open(file_path, "rb") as f:
+                        data = tomllib.load(f)
+                except (IOError, tomllib.TOMLDecodeError) as e:
+                    self._logger.W(f"failed to load entity from {file_path}: {e}")
+                    continue
+
+                # Extract entity ID from filename (remove .toml extension)
+                entity_id = file_path.name[:-5]
+
+                # Create and store raw entity data
+                entities[entity_type][entity_id] = data
+
+        entity_counts = {t: len(e) for t, e in entities.items()}
+        self._logger.D(f"count of loaded entities from filesystem: {entity_counts}")
+        return entities

ruyi/ruyipkg/fetch.py

@@ -0,0 +1,271 @@
+import abc
+import mmap
+import os
+import subprocess
+from typing import Any, Final
+
+import requests
+from rich import progress
+
+from ..log import RuyiLogger
+
+ENV_OVERRIDE_FETCHER: Final = "RUYI_OVERRIDE_FETCHER"
+
+
+def _is_url_ftp(url: str) -> bool:
+    return url.lower().startswith("ftp://")
+
+
+class BaseFetcher:
+    def __init__(self, logger: RuyiLogger, urls: list[str], dest: str) -> None:
+        self._logger = logger
+        self.urls = urls
+        self.dest = dest
+
+    @classmethod
+    @abc.abstractmethod
+    def is_available(cls, logger: RuyiLogger) -> bool:
+        return False
+
+    @abc.abstractmethod
+    def fetch_one(self, url: str, dest: str, resume: bool) -> bool:
+        return False
+
+    def fetch_one_with_retry(
+        self,
+        url: str,
+        dest: str,
+        resume: bool,
+        retries: int,
+    ) -> bool:
+        for t in range(retries):
+            if t > 0:
+                self._logger.I(f"retrying download ({t + 1} of {retries} times)")
+            if self.fetch_one(url, dest, resume):
+                return True
+        return False
+
+    def fetch(self, *, resume: bool = False, retries: int = 3) -> None:
+        for url in self.urls:
+            self._logger.I(f"downloading {url} to {self.dest}")
+            if self.fetch_one_with_retry(url, self.dest, resume, retries):
+                return
+        # all URLs have been tried and all have failed
+        raise RuntimeError(
+            f"failed to fetch '{self.dest}': all source URLs have failed"
+        )
+
+    @classmethod
+    def new(cls, logger: RuyiLogger, urls: list[str], dest: str) -> "BaseFetcher":
+        return get_usable_fetcher_cls(logger)(logger, urls, dest)
+
+
+KNOWN_FETCHERS: Final[dict[str, type[BaseFetcher]]] = {}
+
+
+def register_fetcher(name: str, f: type[BaseFetcher]) -> None:
+    # NOTE: can add priority support if needed
+    KNOWN_FETCHERS[name] = f
+
+
+_fetcher_cache_populated: bool = False
+_cached_usable_fetcher_class: type[BaseFetcher] | None = None
+
+
+def get_usable_fetcher_cls(logger: RuyiLogger) -> type[BaseFetcher]:
+    global _fetcher_cache_populated
+    global _cached_usable_fetcher_class
+
+    if _fetcher_cache_populated:
+        if _cached_usable_fetcher_class is None:
+            raise RuntimeError("no fetcher is available on the system")
+        return _cached_usable_fetcher_class
+
+    _fetcher_cache_populated = True
+
+    if override_name := os.environ.get(ENV_OVERRIDE_FETCHER):
+        logger.D(f"forcing fetcher '{override_name}'")
+
+        cls = KNOWN_FETCHERS.get(override_name)
+        if cls is None:
+            raise RuntimeError(f"unknown fetcher '{override_name}'")
+        if not cls.is_available(logger):
+            raise RuntimeError(
+                f"the requested fetcher '{override_name}' is unavailable on the system"
+            )
+        _cached_usable_fetcher_class = cls
+        return cls
+
+    for name, cls in KNOWN_FETCHERS.items():
+        if not cls.is_available(logger):
+            logger.D(f"fetcher '{name}' is unavailable")
+            continue
+        _cached_usable_fetcher_class = cls
+        return cls
+
+    raise RuntimeError("no fetcher is available on the system")
+
+
+class CurlFetcher(BaseFetcher):
+    def __init__(self, logger: RuyiLogger, urls: list[str], dest: str) -> None:
+        super().__init__(logger, urls, dest)
+
+    @classmethod
+    def is_available(cls, logger: RuyiLogger) -> bool:
+        # try running "curl --version" and it should succeed
+        try:
+            retcode = subprocess.call(["curl", "--version"], stdout=subprocess.DEVNULL)
+            return retcode == 0
+        except Exception as e:
+            logger.D("exception occurred when trying to curl --version:", e)
+            return False
+
+    def fetch_one(self, url: str, dest: str, resume: bool) -> bool:
+        argv = ["curl"]
+        if resume:
+            argv.extend(("-C", "-"))
+
+        # A bug in curl 8.14.1 (and only that version) broke the recognition of
+        # the `--ftp-pasv`` flag, and unfortunately this version is currently
+        # provided by some popular distros so far.
+        #
+        # So, for the vast majority of non-FTP downloads to work even with
+        # this buggy version, we simply do not pass the flag if the URL is
+        # not an FTP one.
+        #
+        # See: https://github.com/curl/curl/issues/17545
+        # See: https://github.com/ruyisdk/ruyi/issues/316
+        if _is_url_ftp(url):
+            argv.append("--ftp-pasv")
+
+        argv.extend(
+            (
+                "-L",
+                "--connect-timeout",
+                "60",
+                "-o",
+                dest,
+                url,
+            )
+        )
+
+        retcode = subprocess.call(argv)
+        if retcode != 0:
+            self._logger.W(
+                f"failed to fetch distfile: command '{' '.join(argv)}' returned {retcode}"
+            )
+            return False
+
+        return True
+
+
+register_fetcher("curl", CurlFetcher)
+
+
+class WgetFetcher(BaseFetcher):
+    def __init__(self, logger: RuyiLogger, urls: list[str], dest: str) -> None:
+        super().__init__(logger, urls, dest)
+
+    @classmethod
+    def is_available(cls, logger: RuyiLogger) -> bool:
+        # try running "wget --version" and it should succeed
+        try:
+            retcode = subprocess.call(["wget", "--version"], stdout=subprocess.DEVNULL)
+            return retcode == 0
+        except Exception as e:
+            logger.D("exception occurred when trying to wget --version:", e)
+            return False
+
+    def fetch_one(self, url: str, dest: str, resume: bool) -> bool:
+        # These arguments are taken from Gentoo
+        argv = ["wget"]
+        if resume:
+            argv.append("-c")
+        # wget does not suffer from the same bug as curl, but to be safe, we
+        # also enable the passive FTP mode only if the URL is an FTP one.
+        if _is_url_ftp(url):
+            argv.append("--passive-ftp")
+        argv.extend(("-T", "60", "-O", dest, url))
+
+        retcode = subprocess.call(argv)
+        if retcode != 0:
+            self._logger.W(
+                f"failed to fetch distfile: command '{' '.join(argv)}' returned {retcode}"
+            )
+            return False
+
+        return True
+
+
+register_fetcher("wget", WgetFetcher)
+
+
+class PythonRequestsFetcher(BaseFetcher):
+    def __init__(self, logger: RuyiLogger, urls: list[str], dest: str) -> None:
+        super().__init__(logger, urls, dest)
+
+        self.chunk_size = 4 * mmap.PAGESIZE
+        # TODO: User-Agent
+
+    @classmethod
+    def is_available(cls, logger: RuyiLogger) -> bool:
+        return True
+
+    def fetch_one(self, url: str, dest: str, resume: bool) -> bool:
+        self._logger.D(f"downloading [cyan]{url}[/] to [cyan]{dest}")
+
+        open_mode = "ab" if resume else "wb"
+        start_from = 0
+        headers: dict[str, str] = {}
+        if resume:
+            filesize = os.stat(dest).st_size
+            self._logger.D(f"resuming from position {filesize}")
+            start_from = filesize
+            headers["Range"] = f"bytes={filesize}-"
+
+        r = requests.get(url, headers=headers, stream=True)
+        total_len: int | None = None
+        if total_len_str := r.headers.get("Content-Length"):
+            total_len = int(total_len_str) + start_from
+
+        try:
+            trc = progress.TimeRemainingColumn(compact=True, elapsed_when_finished=True)  # type: ignore[call-arg,unused-ignore]
+        except TypeError:
+            # rich < 12.0.0 does not support the styles we're asking here, so
+            # just downgrade UX in favor of basic usability in that case.
+            #
+            # see https://github.com/Textualize/rich/pull/1992
+            trc = progress.TimeRemainingColumn()
+
+        columns = (
+            progress.SpinnerColumn(),
+            progress.BarColumn(),
+            progress.DownloadColumn(),
+            progress.TransferSpeedColumn(),
+            trc,
+        )
+        dest_filename = os.path.basename(dest)
+        with open(dest, open_mode) as f:
+            with progress.Progress(*columns, console=self._logger.log_console) as pg:
+                indeterminate = total_len is None
+                kwargs: dict[str, Any]
+                if indeterminate:
+                    # be compatible with rich <= 12.3.0 where add_task()'s `total`
+                    # parameter cannot be None
+                    # see https://github.com/Textualize/rich/commit/052b15785876ad85
+                    kwargs = {"start": False}
+                else:
+                    kwargs = {"total": total_len}
+
+                task = pg.add_task(dest_filename, completed=start_from, **kwargs)
+                for chunk in r.iter_content(self.chunk_size):
+                    f.write(chunk)
+                    # according to the docs it's probably not okay to pulse the
+                    # progress bar if the total number of steps is not yet known
+                    if not indeterminate:
+                        pg.advance(task, len(chunk))
+
+        return True
+
+
+register_fetcher("requests", PythonRequestsFetcher)

ruyi/ruyipkg/host.py

@@ -0,0 +1,55 @@
+import platform
+import sys
+from typing import NamedTuple
+
+
+class RuyiHost(NamedTuple):
+    os: str
+    arch: str
+
+    def __str__(self) -> str:
+        return f"{self.os}/{self.arch}"
+
+    def canonicalize(self) -> "RuyiHost":
+        return RuyiHost(
+            os=canonicalize_os_str(self.os),
+            arch=canonicalize_arch_str(self.arch),
+        )
+
+
+def canonicalize_host_str(host: str | RuyiHost) -> str:
+    if isinstance(host, str):
+        frags = host.split("/", 1)
+        os = "linux" if len(frags) == 1 else frags[0]
+        arch = frags[0] if len(frags) == 1 else frags[1]
+        return str(RuyiHost(os, arch).canonicalize())
+
+    return str(host.canonicalize())
+
+
+def canonicalize_arch_str(arch: str) -> str:
+    # Information sources:
+    #
+    # * https://bugs.python.org/issue7146#msg94134
+    # * https://superuser.com/questions/305901/possible-values-of-processor-architecture
+    match arch.lower():
+        case "amd64" | "em64t":
+            return "x86_64"
+        case "arm64":
+            return "aarch64"
+        case "x86":
+            return "i686"
+        case arch_lower:
+            return arch_lower
+
+
+def canonicalize_os_str(os: str) -> str:
+    match os:
+        case "win32":
+            return "windows"
+        case _:
+            return os
+
+
+def get_native_host() -> RuyiHost:
+    return RuyiHost(os=sys.platform, arch=platform.machine()).canonicalize()