dyp 0.2.0__py3-none-any.whl

dyp/__init__.py

@@ -0,0 +1,21 @@
+"""dyp: a dnf5-style package manager for Python wheels. No pip involved.
+
+Scriptable API::
+
+    import dyp
+    env = dyp.Env.load()                       # or Env.load("/path/to/venv/bin/python")
+    index = dyp.Index(env)                     # PyPI client (cached, parallel, PEP 658)
+    pkgs = dyp.resolve(["requests"], env=env, index=index)
+    dyp.Session(pkgs, on_event=print).run()    # parallel, resumable, sha256-verified
+    for p in pkgs:
+        dyp.install_wheel(p.path, env, requested=p.requested)
+"""
+__version__ = "0.2.0"
+
+from .backend import DypError, Package, cache_dir  # noqa: F401
+from .download import Event, Session  # noqa: F401
+from .env import Env  # noqa: F401
+from .index import Index, ProjectNotFound  # noqa: F401
+from .planner import check_updates, resolve  # noqa: F401
+from .ui import DownloadView, Ui  # noqa: F401
+from .wheel import Dist, install_wheel, scan, uninstall  # noqa: F401

dyp/__main__.py

@@ -0,0 +1,4 @@
+import sys
+from .cli import main
+
+sys.exit(main())

dyp/_http.py

@@ -0,0 +1,95 @@
+"""Tiny HTTP client: keep-alive connection per thread, gzip, redirects, proxy fallback."""
+from __future__ import annotations
+
+import gzip
+import http.client
+import os
+import threading
+import urllib.parse
+import urllib.request
+from urllib.parse import urljoin, urlsplit
+
+from .backend import UA, DypError
+
+
+class Http:
+    def __init__(self):
+        self.local = threading.local()
+        self.proxies = urllib.request.getproxies()
+
+    def _proxy(self, scheme, netloc):
+        """(host, port, auth headers) of the HTTPS proxy to tunnel through, or None."""
+        url = self.proxies.get(scheme)
+        if not url or scheme != "https" or urllib.request.proxy_bypass(netloc.split(":")[0]):
+            return None
+        p = urlsplit(url if "//" in url else "//" + url)
+        hdr = {}
+        if p.username:
+            import base64
+            tok = base64.b64encode(f"{urllib.parse.unquote(p.username)}:"
+                                   f"{urllib.parse.unquote(p.password or '')}".encode()).decode()
+            hdr["Proxy-Authorization"] = f"Basic {tok}"
+        return p.hostname, p.port or 8080, hdr
+
+    def _conn(self, scheme, netloc):
+        conns = self.local.__dict__.setdefault("c", {})
+        c = conns.get((scheme, netloc))
+        if c is None:
+            proxy = self._proxy(scheme, netloc)
+            if proxy:                                     # keep-alive *through* the proxy (CONNECT)
+                c = http.client.HTTPSConnection(proxy[0], proxy[1], timeout=30)
+                host, _, port = netloc.partition(":")
+                c.set_tunnel(host, int(port) if port else 443, headers=proxy[2])
+            elif scheme == "https":
+                c = http.client.HTTPSConnection(netloc, timeout=30)
+            else:
+                c = http.client.HTTPConnection(netloc, timeout=30)
+            conns[(scheme, netloc)] = c
+        return c
+
+    def _drop(self, scheme, netloc):
+        c = self.local.__dict__.get("c", {}).pop((scheme, netloc), None)
+        if c:
+            c.close()
+
+    def get(self, url, headers=None, method="GET"):
+        """-> (status, lower-cased headers, body bytes). Follows up to 5 redirects."""
+        hdr = {"User-Agent": UA, "Accept-Encoding": "gzip", **(headers or {})}
+        for _ in range(6):
+            if urlsplit(url).scheme == "http" and self.proxies.get("http"):
+                req = urllib.request.Request(url, headers=hdr, method=method)
+                try:
+                    r = urllib.request.urlopen(req, timeout=30)
+                    status, h, body = r.status, {k.lower(): v for k, v in r.headers.items()}, r.read()
+                except urllib.error.HTTPError as e:
+                    status, h, body = e.code, {k.lower(): v for k, v in e.headers.items()}, e.read()
+                except OSError as e:
+                    raise DypError(f"network error for {url}: {e}") from e
+            else:
+                u = urlsplit(url)
+                path = (u.path or "/") + (f"?{u.query}" if u.query else "")
+                for attempt in (0, 1):
+                    c = self._conn(u.scheme, u.netloc)
+                    try:
+                        c.request(method, path, headers=hdr)
+                        r = c.getresponse()
+                        body = r.read()
+                        break
+                    except (OSError, http.client.HTTPException) as e:
+                        self._drop(u.scheme, u.netloc)
+                        if attempt:
+                            raise DypError(f"network error for {url}: {e}") from e
+                status, h = r.status, {k.lower(): v for k, v in r.getheaders()}
+            if status in (301, 302, 303, 307, 308) and h.get("location"):
+                url = urljoin(url, h["location"])
+                continue
+            if h.get("content-encoding") == "gzip":
+                body = gzip.decompress(body)
+            return status, h, body
+        raise DypError(f"too many redirects for {url}")
+
+    def download(self, url, dest, chunk=1 << 16):
+        req = urllib.request.Request(url, headers={"User-Agent": UA})
+        with urllib.request.urlopen(req, timeout=30) as r, open(dest, "wb") as f:
+            while b := r.read(chunk):
+                f.write(b)

dyp/backend.py

@@ -0,0 +1,88 @@
+"""Shared types and helpers. dyp no longer shells out to pip for anything."""
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import unquote
+
+UA = "dyp/0.2 (+https://pypi.org/project/dyp/)"
+PROTECTED = {"pip", "dyp", "packaging"}
+
+
+class DypError(Exception):
+    pass
+
+
+def cache_root() -> Path:
+    if os.name == "nt":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    return base / "dyp"
+
+
+def cache_dir(sub: str = "packages") -> Path:
+    d = cache_root() / sub
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def default_python() -> str | None:
+    """Target the *active* virtualenv even if dyp itself lives elsewhere (like pip's `python -m pip`
+    would, but also for a globally installed `dyp` command)."""
+    venv = os.environ.get("VIRTUAL_ENV")
+    if venv and os.path.realpath(sys.prefix) != os.path.realpath(venv):
+        cand = Path(venv) / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+        if cand.exists():
+            return str(cand)
+    return None
+
+
+@dataclass
+class Package:
+    name: str
+    version: str
+    url: str
+    sha256: str | None
+    requested: bool = False
+    size: int = 0
+    repo: str = "pypi"
+    old_version: str | None = None     # installed version this package replaces
+    old_size: int = 0
+    old_path: str | None = None        # its .dist-info directory
+
+    @property
+    def filename(self) -> str:
+        return unquote(self.url.split("#")[0].rsplit("/", 1)[-1])
+
+    @property
+    def is_wheel(self) -> bool:
+        return self.filename.endswith(".whl")
+
+    @property
+    def arch(self) -> str:
+        if not self.is_wheel:
+            return "src"
+        plat = self.filename[:-4].split("-")[-1]
+        if plat == "any":
+            return "noarch"
+        for key, arch in (("x86_64", "x86_64"), ("amd64", "x86_64"), ("aarch64", "aarch64"),
+                          ("arm64", "arm64"), ("universal2", "universal2"), ("win32", "i686"),
+                          ("i686", "i686"), ("ppc64le", "ppc64le"), ("s390x", "s390x")):
+            if key in plat:
+                return arch
+        return plat.split(".")[0]
+
+    @property
+    def nevra(self) -> str:
+        return f"{self.name}-{self.version}.{self.arch}"
+
+    @property
+    def path(self) -> Path:
+        return cache_dir() / self.filename
+
+    @property
+    def cached(self) -> bool:
+        return self.path.exists() and (not self.size or self.path.stat().st_size == self.size)

dyp/cli.py

@@ -0,0 +1,311 @@
+"""dyp command line: a dnf5-flavoured package manager for Python wheels."""
+from __future__ import annotations
+
+import argparse
+import shutil
+import sys
+
+from packaging.utils import canonicalize_name
+from packaging.version import Version
+
+from . import __version__
+from . import planner
+from .backend import PROTECTED, DypError, cache_dir, cache_root, default_python
+from .download import Session, sha256_of
+from .env import Env
+from .index import Index
+from .ui import DownloadView, Ui, tsz, whole
+from .wheel import install_wheel, scan, uninstall
+
+
+def _label(index: Index) -> str:
+    return " PyPI - pypi.org" if index.host == "pypi.org" else f" {index.host}"
+
+
+def _load(a, ui):
+    env = Env.load(a.python or default_python(), a.user)
+    return env, Index(env, getattr(a, "index_url", None))
+
+
+# ---------------------------------------------------------------- shared flow
+def _transaction(a, ui: Ui, env: Env, pkgs) -> int:
+    """Show the plan, ask, download, then install natively."""
+    new_req = [p for p in pkgs if p.old_version is None and p.requested]
+    new_dep = [p for p in pkgs if p.old_version is None and not p.requested]
+    ups = [p for p in pkgs if p.old_version and Version(p.version) > Version(p.old_version)]
+    downs = [p for p in pkgs if p.old_version and Version(p.version) < Version(p.old_version)]
+    same = [p for p in pkgs if p.old_version and Version(p.version) == Version(p.old_version)]
+    ups += same
+
+    def row(p):
+        return (p.name, p.arch, p.version, p.repo, tsz(p.size))
+
+    def row_rep(p):
+        return (row(p), [(p.name, p.arch, p.old_version, "@System", tsz(p.old_size))])
+
+    sections = []
+    for title, grp, fn in (("Installing:", new_req, row), ("Installing dependencies:", new_dep, row),
+                           ("Upgrading:", ups, row_rep), ("Downgrading:", downs, row_rep)):
+        if grp:
+            sections.append((title, [fn(p) for p in grp]))
+    ui.table(sections)
+    ui.summary([(l, len(g)) for l, g in (("Installing", new_req + new_dep), ("Upgrading", ups),
+                                         ("Downgrading", downs)) if g])
+    to_get = [p for p in pkgs if not p.cached]
+    inbound, need = sum(p.size for p in pkgs), sum(p.size for p in to_get)
+    ui.print(f"\nTotal size of inbound packages is {whole(inbound)}. Need to download {whole(need)}.")
+
+    if a.dry_run:
+        ui.print("Dry run, nothing was changed.")
+        return 0
+    if not ui.confirm(a.yes):
+        ui.print("Operation aborted by the user.")
+        return 1
+
+    if to_get:
+        view = DownloadView(ui, to_get)
+        Session(to_get, parallel=a.parallel, on_event=view).run()
+        view.close()
+        if view.failed:
+            for name, e in view.failed:
+                ui.print(f"Failed to download {name}: {e}")
+            return 1
+
+    def verify():
+        for p in pkgs:
+            if p.sha256 and sha256_of(p.path) != p.sha256:
+                raise DypError(f"{p.filename}: checksum mismatch (try `dyp clean`)")
+
+    installed = scan(env)
+
+    def one(p):
+        old = installed.get(p.name)
+        return lambda report: install_wheel(p.path, env, requested=p.requested, replace=old, progress=report)
+
+    ui.print("Running transaction")
+    steps = [("Verify package files", verify, inbound or len(pkgs)),
+             ("Prepare transaction", lambda: None, len(pkgs))]
+    for p in pkgs:
+        verb = "Installing" if p.old_version is None else \
+            "Upgrading" if Version(p.version) >= Version(p.old_version) else "Downgrading"
+        steps.append((f"{verb} {p.nevra}", one(p), p.size, True))
+    try:
+        ui.steps(steps)
+    except DypError as e:
+        ui.print(f"Transaction failed: {e}")
+        return 1
+    ui.print("Complete!")
+    return 0
+
+
+def _plan(a, ui, env, index, **kw):
+    """Run the 'loading repositories' phase; returns (pkgs, error)."""
+    ui.print("Updating and loading repositories:")
+
+    def work():
+        try:
+            pkgs = planner.resolve(env=env, index=index, pre=a.pre, **kw)
+            return (pkgs, None), index.bytes
+        except DypError as e:
+            return (None, e), index.bytes
+
+    pkgs, err = ui.run_row(_label(index), work)
+    ui.print("Repositories loaded.")
+    if err:
+        ui.print("Failed to resolve the transaction:")
+        ui.print(str(err))
+    return pkgs, err
+
+
+# ------------------------------------------------------------------- commands
+def cmd_install(a, ui: Ui) -> int:
+    env, index = _load(a, ui)
+    pkgs, err = _plan(a, ui, env, index, specs=a.packages, requirements=a.requirement,
+                      upgrade=a.upgrade)
+    if err:
+        return 1
+    if not pkgs:
+        inst = scan(env)
+        for spec in a.packages:
+            from .resolver import parse_req
+            d = inst.get(canonicalize_name(parse_req(spec).name))
+            ui.print(f'Package "{d.key}-{d.version}" is already installed.' if d else f'Package "{spec}" is already installed.')
+        ui.print("Nothing to do.")
+        return 0
+    return _transaction(a, ui, env, pkgs)
+
+
+def cmd_upgrade(a, ui: Ui) -> int:
+    env, index = _load(a, ui)
+    if a.packages:
+        inst = scan(env)
+        missing = [p for p in a.packages if canonicalize_name(p) not in inst]
+        if missing:
+            ui.print("Failed to resolve the transaction:")
+            for m in missing:
+                ui.print(f"No match for argument: {m}")
+            return 1
+        kw = dict(specs=a.packages, upgrade=True)
+    else:
+        kw = dict(upgrade_all=True)
+    pkgs, err = _plan(a, ui, env, index, requirements=[], **kw)
+    if err:
+        return 1
+    if not pkgs:
+        ui.print("Nothing to do.")
+        return 0
+    return _transaction(a, ui, env, pkgs)
+
+
+def cmd_check_update(a, ui: Ui) -> int:
+    env, index = _load(a, ui)
+    ui.print("Updating and loading repositories:")
+    found = ui.run_row(_label(index), lambda: (planner.check_updates(env, index, pre=a.pre), index.bytes))
+    ui.print("Repositories loaded.")
+    if not found:
+        ui.print("Nothing to do.")
+        return 0
+    ui.print("")
+    w = max(len(d.name) for d, _ in found)
+    for d, v in found:
+        ui.print(f"{ui.paint('32', d.name)}{' ' * (w - len(d.name))}  {str(v):<14} {index.host if index.host != 'pypi.org' else 'pypi'}")
+    return 100                                    # like dnf: 100 = updates available
+
+
+def cmd_remove(a, ui: Ui) -> int:
+    env = Env.load(a.python or default_python(), a.user)
+    inst = scan(env)
+    ui.print("Updating and loading repositories:")
+    ui.run_row(" @System", lambda: (None, 0))
+    ui.print("Repositories loaded.")
+    dists, missing = [], []
+    for q in a.packages:
+        d = inst.get(canonicalize_name(q))
+        (dists if d else missing).append(d or q)
+    if missing:
+        ui.print("Failed to resolve the transaction:")
+        for m in missing:
+            ui.print(f"No match for argument: {m}")
+        return 1
+    prot = [d.name for d in dists if d.key in PROTECTED]
+    if prot:
+        ui.print("Failed to resolve the transaction:")
+        ui.print(f"The operation would result in removing protected packages: {', '.join(prot)}")
+        return 1
+    ui.table([("Removing:", [(d.name, "noarch", d.version, "@System", tsz(d.size)) for d in dists])],
+             green="31")
+    freed = sum(d.size for d in dists)
+    ui.summary([("Removing", len(dists))])
+    ui.print(f"\nAfter this operation, {whole(freed)} will be freed (install 0 B, remove {whole(freed)}).")
+    if a.dry_run:
+        ui.print("Dry run, nothing was removed.")
+        return 0
+    if not ui.confirm(a.yes):
+        ui.print("Operation aborted by the user.")
+        return 1
+
+    def rm(d):
+        return lambda report: uninstall(d, env, lambda i, n: report(int(d.size * i / max(n, 1)), d.size))
+
+    ui.print("Running transaction")
+    steps = [("Prepare transaction", lambda: None, len(dists))]
+    steps += [(f"Removing {d.key}-{d.version}", rm(d), d.size, True) for d in dists]
+    try:
+        ui.steps(steps)
+    except DypError as e:
+        ui.print(f"Transaction failed: {e}")
+        return 1
+    ui.print("Complete!")
+    return 0
+
+
+def cmd_list(a, ui: Ui) -> int:
+    env = Env.load(a.python or default_python(), a.user)
+    pkgs = sorted(scan(env).values(), key=lambda d: d.key)
+    if a.filter:
+        pkgs = [d for d in pkgs if any(f.lower() in d.key for f in a.filter)]
+    ui.print("Installed packages")
+    names = [d.name for d in pkgs]
+    w = max(map(len, names), default=0)
+    for d, n in zip(pkgs, names):
+        ui.print(f"{ui.paint('32', n)}{' ' * (w - len(n))}  {d.version}")
+    return 0
+
+
+def cmd_clean(a, ui: Ui) -> int:
+    root = cache_root()
+    files = [f for f in root.rglob("*") if f.is_file()] if root.exists() else []
+    freed = sum(f.stat().st_size for f in files)
+    shutil.rmtree(root, ignore_errors=True)
+    ui.print(f"{len(files)} files removed, {whole(freed)} freed.")
+    return 0
+
+
+# ------------------------------------------------------------------- parser
+def build_parser():
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("-y", "--yes", action="store_true", help="answer yes to the prompt")
+    common.add_argument("--python", help="target interpreter / venv python (default: active venv or this Python)")
+    common.add_argument("--user", action="store_true", help="user site instead of the environment")
+    common.add_argument("--dry-run", action="store_true", help="show the transaction, change nothing")
+    net = argparse.ArgumentParser(add_help=False)
+    net.add_argument("--pre", action="store_true", help="allow pre-releases")
+    net.add_argument("--index-url", help="package index (PEP 503/691), default PyPI")
+    net.add_argument("--parallel", type=int, default=3, help="simultaneous downloads (default 3)")
+
+    p = argparse.ArgumentParser(prog="dyp", description="dnf5-style package manager for Python wheels")
+    p.add_argument("--version", action="version", version=f"dyp {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    i = sub.add_parser("install", aliases=["in", "add"], parents=[common, net], help="install packages")
+    i.add_argument("packages", nargs="*", help="names or requirement specifiers")
+    i.add_argument("-r", "--requirement", action="append", default=[], help="requirements file")
+    i.add_argument("-U", "--upgrade", action="store_true", help="upgrade the named packages")
+    i.set_defaults(fn=cmd_install)
+
+    u = sub.add_parser("upgrade", aliases=["up", "update"], parents=[common, net],
+                       help="upgrade packages (all of them when none are named)")
+    u.add_argument("packages", nargs="*")
+    u.set_defaults(fn=cmd_upgrade)
+
+    c = sub.add_parser("check-update", aliases=["cu"], parents=[common, net], help="list available updates")
+    c.set_defaults(fn=cmd_check_update)
+
+    r = sub.add_parser("remove", aliases=["rm", "erase"], parents=[common], help="remove packages")
+    r.add_argument("packages", nargs="+")
+    r.set_defaults(fn=cmd_remove)
+
+    l = sub.add_parser("list", aliases=["ls"], parents=[common], help="list installed packages")
+    l.add_argument("filter", nargs="*")
+    l.set_defaults(fn=cmd_list)
+
+    k = sub.add_parser("clean", help="delete the download/index cache")
+    k.set_defaults(fn=cmd_clean)
+    return p
+
+
+def main(argv=None) -> int:
+    for stream in (sys.stdout, sys.stderr):          # cp1252 consoles must never crash us
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except (AttributeError, ValueError):
+            pass
+    a = build_parser().parse_args(argv)
+    if a.cmd in ("install", "in", "add") and not (a.packages or a.requirement):
+        print("dyp install: give at least one package or -r requirements.txt", file=sys.stderr)
+        return 2
+    ui = Ui()
+    try:
+        return a.fn(a, ui)
+    except DypError as e:
+        ui.print(f"Error: {e}")
+        return 1
+    except KeyboardInterrupt:
+        ui.print("\nOperation aborted by the user.")
+        return 130
+    finally:
+        ui.show()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dyp/download.py

@@ -0,0 +1,89 @@
+"""Parallel, resumable, checksum-verified downloads that report through events."""
+from __future__ import annotations
+
+import hashlib
+import os
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Callable
+
+from .backend import UA, DypError, Package
+
+
+@dataclass
+class Event:
+    kind: str                 # "start" | "progress" | "done" | "error"
+    name: str                 # Package.filename
+    done: int = 0
+    total: int = 0
+    rate: float = 0.0
+    error: str | None = None
+
+
+def sha256_of(path, chunk=1 << 20) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while b := f.read(chunk):
+            h.update(b)
+    return h.hexdigest()
+
+
+class Session:
+    def __init__(self, packages, *, parallel=3, on_event: Callable[[Event], None] | None = None):
+        self.packages = list(packages)
+        self.parallel = parallel
+        self.emit = on_event or (lambda e: None)
+        self.failed: list[Package] = []
+
+    def _fetch(self, p: Package, retry=True):
+        path = p.path
+        part = path.with_name(path.name + ".part")
+        have = part.stat().st_size if part.exists() else 0
+        headers = {"User-Agent": UA}
+        if have:
+            headers["Range"] = f"bytes={have}-"
+        try:
+            try:
+                r = urllib.request.urlopen(urllib.request.Request(p.url, headers=headers), timeout=30)
+            except urllib.error.HTTPError as e:
+                if e.code == 416 and retry:            # stale .part
+                    part.unlink(missing_ok=True)
+                    return self._fetch(p, retry=False)
+                raise
+            with r:
+                resumed = r.status == 206
+                if not resumed:
+                    have = 0
+                total = p.size or have + int(r.headers.get("Content-Length", 0))
+                self.emit(Event("start", p.filename, have, total))
+                got, t0, last = have, time.monotonic(), 0.0
+                with open(part, "ab" if resumed else "wb") as f:
+                    while chunk := r.read(64 * 1024):
+                        f.write(chunk)
+                        got += len(chunk)
+                        now = time.monotonic()
+                        if now - last >= 0.05:
+                            last = now
+                            self.emit(Event("progress", p.filename, got, total,
+                                            (got - have) / max(now - t0, 1e-6)))
+            if p.sha256 and sha256_of(part) != p.sha256:
+                part.unlink(missing_ok=True)
+                raise DypError("checksum mismatch")
+            os.replace(part, path)
+            self.emit(Event("done", p.filename, got, total or got))
+        except Exception as e:
+            self.failed.append(p)
+            self.emit(Event("error", p.filename, error=str(e)))
+
+    def run(self) -> list[Package]:
+        pool = ThreadPoolExecutor(self.parallel)
+        try:
+            list(pool.map(self._fetch, self.packages))
+        except KeyboardInterrupt:
+            pool.shutdown(wait=False, cancel_futures=True)
+            raise
+        pool.shutdown()
+        return self.failed