modsmith 0.1.0__py3-none-any.whl

modsmith/net/_resource.py

@@ -0,0 +1,118 @@
+"""Download targets and hash checks."""
+
+import hashlib
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import IO
+
+HASH_PRIORITY = ("sha512", "sha256", "sha1", "md5")
+
+
+def file_hash(path: Path, algorithm: str = "sha1") -> str:
+    """Hash a file without loading it all into memory.
+
+    Args:
+        path: The file.
+        algorithm: A name that `hashlib.new` accepts.
+
+    Returns:
+        The hexadecimal digest.
+    """
+    with open(path, "rb") as file:
+        return stream_hash(file, algorithm)
+
+
+def stream_hash(file: IO[bytes], algorithm: str = "sha1") -> str:
+    """Hash an open file without loading it all into memory.
+
+    Args:
+        file: The open file.
+        algorithm: A name that `hashlib.new` accepts.
+
+    Returns:
+        The hexadecimal digest.
+    """
+    digest = hashlib.new(algorithm)
+    for chunk in iter(lambda: file.read(1 << 20), b""):
+        digest.update(chunk)
+    return digest.hexdigest()
+
+
+def strongest_hash(hashes: Mapping[str, str]) -> tuple[str, str] | None:
+    """Pick the strongest known hash.
+
+    Args:
+        hashes: Digests keyed by algorithm name.
+
+    Returns:
+        The algorithm and its lowercase digest, or None if none is known.
+    """
+    for algorithm in HASH_PRIORITY:
+        value = hashes.get(algorithm)
+        if value:
+            return algorithm, value.lower()
+    return None
+
+
+def matches(path: Path, hashes: Mapping[str, str]) -> bool:
+    """Check a file against its strongest hash.
+
+    Args:
+        path: The file.
+        hashes: Digests keyed by algorithm name.
+
+    Returns:
+        True if the digest matches, or if no known hash was given.
+    """
+    expected = strongest_hash(hashes)
+    return expected is None or file_hash(path, expected[0]) == expected[1]
+
+
+@dataclass(frozen=True)
+class Resource:
+    """A file to download: where from, where to, and how to verify it.
+
+    Attributes:
+        urls: Download locations, tried in order.
+        path: The destination.
+        hashes: Expected digests keyed by algorithm name.
+        size: The expected size in bytes, or None if unknown.
+    """
+
+    urls: tuple[str, ...]
+    path: Path
+    hashes: Mapping[str, str] = field(default_factory=dict[str, str], hash=False)
+    size: int | None = None
+
+    @classmethod
+    def of(
+        cls,
+        url: str,
+        path: Path,
+        hashes: Mapping[str, str] | None = None,
+        size: int | None = None,
+    ) -> "Resource":
+        """Create a resource with one URL.
+
+        Args:
+            url: The download location.
+            path: The destination.
+            hashes: Expected digests keyed by algorithm name.
+            size: The expected size in bytes, or None if unknown.
+
+        Returns:
+            The resource.
+        """
+        return cls((url,), path, dict(hashes or {}), size)
+
+    def is_present(self) -> bool:
+        """Return whether the destination already passes the size and hash checks.
+
+        With no size or hashes to check, any existing file counts as present.
+        """
+        if not self.path.is_file():
+            return False
+        if self.size is not None and self.path.stat().st_size != self.size:
+            return False
+        return matches(self.path, self.hashes)

modsmith/net/_session.py

@@ -0,0 +1,241 @@
+"""HTTP session with retries, shared by everything that uses the network."""
+
+import threading
+import time
+from collections.abc import Collection, Generator, Mapping
+from contextlib import contextmanager
+from importlib.metadata import PackageNotFoundError, version
+from types import TracebackType
+from typing import Any, Self
+
+import requests
+
+from ..errors import NetworkError
+
+try:
+    __version__ = version("modsmith")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+DEFAULT_USER_AGENT = f"modsmith/{__version__} (https://github.com/PaPa-jun/ModSmith)"
+DEFAULT_TIMEOUT = (10.0, 60.0)
+MAX_RETRY_AFTER = 60.0
+RETRY_STATUS = frozenset({429, 500, 502, 503, 504})
+TRANSIENT_ERRORS = (
+    requests.ConnectionError,
+    requests.Timeout,
+    requests.exceptions.ChunkedEncodingError,
+)
+
+
+def is_retryable(error: Exception) -> bool:
+    """Return whether a failed request is worth repeating.
+
+    Connection errors, timeouts, and HTTP 429 and 5xx responses are transient.
+    """
+    if isinstance(error, NetworkError):
+        return error.status is None or error.status in RETRY_STATUS
+    return isinstance(error, TRANSIENT_ERRORS)
+
+
+@contextmanager
+def session_scope(session: "Session | None") -> Generator["Session"]:
+    """Yield ``session``, or a temporary `Session` closed on exit."""
+    if session is not None:
+        yield session
+        return
+    with Session() as owned:
+        yield owned
+
+
+def retry_delay(headers: Mapping[str, str]) -> float | None:
+    """Return the delay a ``Retry-After`` header asks for, capped at one minute.
+
+    Args:
+        headers: The response headers.
+
+    Returns:
+        The delay in seconds, or None if there is no delay in seconds.
+    """
+    value = headers.get("Retry-After", "")
+    return min(float(value), MAX_RETRY_AFTER) if value.isdigit() else None
+
+
+class Session:
+    """HTTP client with retries. Safe to share between threads.
+
+    Each thread gets its own connection pool. Use it as a context manager, or
+    call `close` when you are done.
+
+    Args:
+        user_agent: The ``User-Agent`` header sent with every request.
+        attempts: How many times to try each request.
+        backoff: Seconds to wait before the first retry. Doubles on each retry.
+        timeout: The connect and read timeouts in seconds.
+    """
+
+    def __init__(
+        self,
+        user_agent: str = DEFAULT_USER_AGENT,
+        attempts: int = 4,
+        backoff: float = 1.0,
+        timeout: tuple[float, float] = DEFAULT_TIMEOUT,
+    ) -> None:
+        self.user_agent = user_agent
+        self.attempts = attempts
+        self.backoff = backoff
+        self.timeout = timeout
+        self._local = threading.local()
+        self._lock = threading.Lock()
+        self._sessions: list[requests.Session] = []
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Close the connections of every thread.
+
+        The session stays usable and reconnects on the next request.
+        """
+        with self._lock:
+            for session in self._sessions:
+                session.close()
+            self._sessions.clear()
+        self._local = threading.local()
+
+    @property
+    def http(self) -> requests.Session:
+        """The calling thread's `requests.Session`, created on first use."""
+        session: requests.Session | None = getattr(self._local, "session", None)
+        if session is None:
+            session = requests.Session()
+            session.headers["User-Agent"] = self.user_agent
+            self._local.session = session
+            with self._lock:
+                self._sessions.append(session)
+        return session
+
+    def wait(self, attempt: int, delay: float | None = None) -> None:
+        """Sleep before a retry.
+
+        Args:
+            attempt: The zero-based index of the retry.
+            delay: Seconds the server asked for, instead of the normal backoff.
+        """
+        time.sleep(self.backoff * 2**attempt if delay is None else delay)
+
+    def request(
+        self, method: str, url: str, *, accept: Collection[int] = (), **kwargs: Any
+    ) -> requests.Response:
+        """Send a request, retrying transient failures.
+
+        Retries stop once a response has been accepted, so reading a streamed
+        body is not covered. A delay in ``Retry-After`` is respected, up to one
+        minute.
+
+        Args:
+            method: The HTTP method.
+            url: The URL.
+            accept: Error statuses to return instead of raising.
+            **kwargs: Passed to `requests.Session.request`.
+
+        Returns:
+            A successful response, or one with a status in ``accept``.
+
+        Raises:
+            NetworkError: The last attempt failed or returned an error status.
+        """
+        kwargs.setdefault("timeout", self.timeout)
+        last = NetworkError(f"{method} {url} was not attempted")
+        delay: float | None = None
+        for attempt in range(self.attempts):
+            if attempt:
+                self.wait(attempt - 1, delay)
+            delay = None
+            try:
+                response = self.http.request(method, url, **kwargs)
+            except requests.RequestException as e:
+                last = NetworkError(f"{method} {url} failed: {e}")
+                if not is_retryable(e):
+                    break
+                continue
+            if response.ok or response.status_code in accept:
+                return response
+            delay = retry_delay(response.headers)
+            response.close()
+            last = NetworkError(
+                f"{method} {url} returned {response.status_code}", response.status_code
+            )
+            if not is_retryable(last):
+                break
+        raise last
+
+    def get_json(self, url: str, **kwargs: Any) -> Any:
+        """Send a GET request and decode the JSON response.
+
+        Args:
+            url: The URL.
+            **kwargs: Passed to `requests.Session.request`.
+
+        Returns:
+            The decoded body.
+
+        Raises:
+            NetworkError: The request failed or the body is not JSON.
+        """
+        return self._json(self.request("GET", url, **kwargs), url)
+
+    def post_json(self, url: str, **kwargs: Any) -> Any:
+        """Send a POST request and decode the JSON response.
+
+        Args:
+            url: The URL.
+            **kwargs: Passed to `requests.Session.request`.
+
+        Returns:
+            The decoded body.
+
+        Raises:
+            NetworkError: The request failed or the body is not JSON.
+        """
+        return self._json(self.request("POST", url, **kwargs), url)
+
+    def stream(
+        self,
+        url: str,
+        headers: Mapping[str, str] | None = None,
+        *,
+        accept: Collection[int] = (),
+    ) -> requests.Response:
+        """Open a streaming GET request, with the same retries as `request`.
+
+        Failures while reading the body are not retried, because the caller
+        has to decide how to resume.
+
+        Args:
+            url: The URL.
+            headers: Extra request headers.
+            accept: Error statuses to return instead of raising.
+
+        Returns:
+            The open response. Use it as a context manager.
+
+        Raises:
+            NetworkError: The last attempt failed or returned an error status.
+        """
+        return self.request("GET", url, accept=accept, headers=headers, stream=True)
+
+    def _json(self, response: requests.Response, url: str) -> Any:
+        """Decode a JSON response, raising `NetworkError` if it is invalid."""
+        try:
+            return response.json()
+        except ValueError as e:
+            raise NetworkError(f"Invalid JSON from {url}") from e

modsmith/packs/__init__.py

@@ -0,0 +1,18 @@
+"""Read modpack archives. `load` detects the format from the archive."""
+
+from ._formats import CurseForgePack, Mrpack, PackFormat, load, register
+from ._model import FileRef, Modpack, PackEntry, Side, Support, safe_relative
+
+__all__ = [
+    "CurseForgePack",
+    "FileRef",
+    "Modpack",
+    "Mrpack",
+    "PackEntry",
+    "PackFormat",
+    "Side",
+    "Support",
+    "load",
+    "register",
+    "safe_relative",
+]

modsmith/packs/_formats.py

@@ -0,0 +1,262 @@
+"""Modpack archive formats, and detecting which one an archive uses."""
+
+import json
+import zipfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, Field, ValidationError
+
+from ..errors import PackError, UnsupportedError
+from ._model import FileRef, Modpack, PackEntry, Support, safe_relative
+
+
+class PackFormat(ABC):
+    """Reads one archive format. Subclass it and `register` it to add another.
+
+    Attributes:
+        name: The format's name.
+        index_file: The file in the archive that identifies this format.
+    """
+
+    name: ClassVar[str]
+    index_file: ClassVar[str]
+
+    @abstractmethod
+    def parse(self, index: dict[str, Any], path: Path) -> Modpack:
+        """Build a `Modpack` from the archive's decoded index file.
+
+        Args:
+            index: The decoded index file.
+            path: The archive.
+
+        Returns:
+            The modpack.
+
+        Raises:
+            PackError: The index is invalid.
+        """
+
+
+_formats: list[PackFormat] = []
+
+
+def register(format_: PackFormat) -> PackFormat:
+    """Add a format. The most recently registered format is tried first.
+
+    Args:
+        format_: The format.
+
+    Returns:
+        The same format.
+    """
+    _formats.insert(0, format_)
+    return format_
+
+
+def load(path: Path | str) -> Modpack:
+    """Read a modpack archive. Needs no network and no API key.
+
+    Args:
+        path: The archive.
+
+    Returns:
+        The modpack. CurseForge packs still need `Platform.resolve`.
+
+    Raises:
+        PackError: The archive is invalid or its format is unknown.
+        UnsupportedError: The pack is for another game.
+    """
+    archive_path = Path(path)
+    try:
+        with zipfile.ZipFile(archive_path) as archive:
+            names = set(archive.namelist())
+            for format_ in _formats:
+                if format_.index_file in names:
+                    index = json.loads(archive.read(format_.index_file))
+                    return format_.parse(index, archive_path)
+    except (OSError, zipfile.BadZipFile, ValueError) as e:
+        raise PackError(f"{archive_path.name} is not a valid modpack: {e}") from e
+    raise PackError(f"{archive_path.name} is not a known modpack format")
+
+
+class _MrpackEnv(BaseModel):
+    """A file's ``env`` object in ``modrinth.index.json``."""
+
+    client: Support = "required"
+    server: Support = "required"
+
+
+class _MrpackFile(BaseModel):
+    """An entry of ``files`` in ``modrinth.index.json``."""
+
+    path: str
+    hashes: dict[str, str]
+    downloads: list[str]
+    file_size: int | None = Field(default=None, alias="fileSize")
+    env: _MrpackEnv = Field(default_factory=_MrpackEnv)
+
+
+class _MrpackIndex(BaseModel):
+    """The contents of ``modrinth.index.json``."""
+
+    format_version: int = Field(alias="formatVersion")
+    game: str
+    name: str
+    version_id: str = Field(alias="versionId")
+    files: list[_MrpackFile]
+    dependencies: dict[str, str]
+
+
+class Mrpack(PackFormat):
+    """The Modrinth ``.mrpack`` format.
+
+    Attributes:
+        roots: The folders that hold overrides, keyed by side.
+    """
+
+    name = "mrpack"
+    index_file = "modrinth.index.json"
+    roots: ClassVar[dict[str, str]] = {
+        "common": "overrides",
+        "client": "client-overrides",
+        "server": "server-overrides",
+    }
+
+    def parse(self, index: dict[str, Any], path: Path) -> Modpack:
+        """Build a `Modpack` from ``modrinth.index.json``.
+
+        Args:
+            index: The decoded index file.
+            path: The archive.
+
+        Returns:
+            The modpack. Its entries are already resolved.
+
+        Raises:
+            PackError: The index is invalid or does not name a game version or loader.
+            UnsupportedError: The pack is for another game.
+        """
+        try:
+            data = _MrpackIndex.model_validate(index)
+        except ValidationError as e:
+            raise PackError(f"Invalid {self.index_file}: {e}") from e
+        if data.game != "minecraft":
+            raise UnsupportedError(f"Unsupported game: {data.game}")
+        dependencies = dict(data.dependencies)
+        minecraft = dependencies.pop("minecraft", None)
+        if minecraft is None:
+            raise PackError("Modpack does not declare a Minecraft version")
+        if not dependencies:
+            raise PackError("Modpack does not declare a mod loader")
+        loader_kind, loader_version = next(iter(dependencies.items()))
+        return Modpack(
+            format=self.name,
+            name=data.name,
+            version=data.version_id,
+            minecraft=minecraft,
+            loader_kind=loader_kind,
+            loader_version=loader_version,
+            entries=tuple(
+                PackEntry(
+                    path=safe_relative(file.path),
+                    urls=tuple(file.downloads),
+                    hashes=file.hashes,
+                    size=file.file_size,
+                    client=file.env.client,
+                    server=file.env.server,
+                )
+                for file in data.files
+            ),
+            archive=path,
+            override_roots=self.roots,
+        )
+
+
+class _CfFile(BaseModel):
+    """An entry of ``files`` in a CurseForge ``manifest.json``."""
+
+    project_id: int = Field(alias="projectID")
+    file_id: int = Field(alias="fileID")
+    required: bool = True
+
+
+class _CfLoader(BaseModel):
+    """An entry of ``modLoaders`` in a CurseForge ``manifest.json``."""
+
+    id: str
+    primary: bool = False
+
+
+class _CfMinecraft(BaseModel):
+    """The ``minecraft`` object in a CurseForge ``manifest.json``."""
+
+    version: str
+    mod_loaders: list[_CfLoader] = Field(alias="modLoaders")
+    recommended_ram: int | None = Field(default=None, alias="recommendedRam")
+
+
+class _CfManifest(BaseModel):
+    """The contents of a CurseForge ``manifest.json``."""
+
+    manifest_type: Literal["minecraftModpack"] = Field(alias="manifestType")
+    name: str = "Modpack"
+    version: str = ""
+    overrides: str = "overrides"
+    minecraft: _CfMinecraft
+    files: list[_CfFile] = Field(default_factory=list[_CfFile])
+
+
+class CurseForgePack(PackFormat):
+    """The CurseForge ``.zip`` format."""
+
+    name = "curseforge"
+    index_file = "manifest.json"
+
+    def parse(self, index: dict[str, Any], path: Path) -> Modpack:
+        """Build a `Modpack` from ``manifest.json``.
+
+        Args:
+            index: The decoded index file.
+            path: The archive.
+
+        Returns:
+            The modpack. Its entries need `CurseForge.resolve`.
+
+        Raises:
+            PackError: The manifest is invalid or does not name a mod loader.
+        """
+        try:
+            data = _CfManifest.model_validate(index)
+        except ValidationError as e:
+            raise PackError(f"Invalid {self.index_file}: {e}") from e
+        loaders = data.minecraft.mod_loaders
+        if not loaders:
+            raise PackError("Modpack does not declare a mod loader")
+        loader = next((item for item in loaders if item.primary), loaders[0])
+        kind, _, version = loader.id.partition("-")
+        if not version:
+            raise PackError(f"Invalid mod loader id: {loader.id}")
+
+        def entry(file: _CfFile) -> PackEntry:
+            support: Support = "required" if file.required else "optional"
+            ref = FileRef(self.name, str(file.project_id), str(file.file_id))
+            return PackEntry(client=support, server=support, ref=ref)
+
+        return Modpack(
+            format=self.name,
+            name=data.name,
+            version=data.version,
+            minecraft=data.minecraft.version,
+            loader_kind=kind,
+            loader_version=version,
+            entries=tuple(entry(file) for file in data.files),
+            archive=path,
+            override_roots={"common": data.overrides},
+            memory_mb=data.minecraft.recommended_ram,
+        )
+
+
+register(CurseForgePack())
+register(Mrpack())