snap-maintainer 0.1.0__py3-none-any.whl

snap_maintainer/__init__.py

@@ -0,0 +1 @@
+"""Discover and maintain published snaps."""

snap_maintainer/__main__.py

@@ -0,0 +1,3 @@
+from snap_maintainer.cli import main
+
+raise SystemExit(main())

snap_maintainer/archive.py

@@ -0,0 +1,143 @@
+"""Ubuntu archive metadata: which binary packages a snap stages, and their source packages.
+
+snapcraft doesn't record staged package versions in the snap any more, so we
+reconstruct *which* packages are in it: the stage-packages plus their
+Depends/Pre-Depends closure, minus what the base snap already ships (craft-parts
+leaves those out too).
+"""
+
+from __future__ import annotations
+
+import lzma
+import re
+from pathlib import Path
+
+from snap_maintainer import debver
+from snap_maintainer.cache import cached_json, fetch
+
+COMPONENTS = ("main", "universe", "restricted", "multiverse")
+POCKETS = ("", "-updates", "-security")
+
+
+def _mirror(arch: str) -> str:
+    if arch in ("amd64", "i386"):
+        return "http://archive.ubuntu.com/ubuntu"
+    return "http://ports.ubuntu.com/ubuntu-ports"
+
+
+def _parse_relations(field: str) -> list[list[str]]:
+    """'a (>= 1) | b, c:any' -> [['a', 'b'], ['c']]"""
+    groups = []
+    for group in field.split(","):
+        alts = [re.split(r"[\s(:]", alt.strip(), maxsplit=1)[0] for alt in group.split("|")]
+        alts = [a for a in alts if a]
+        if alts:
+            groups.append(alts)
+    return groups
+
+
+def parse_packages(text: str, index: dict) -> None:
+    """Merge a Packages file into ``index`` =
+    {pkg: {"source": str, "version": source version, "deps": [[alts]]},
+     "__provides__": {virtual: [pkgs]}}, keeping the newest version of each package."""
+    provides = index.setdefault("__provides__", {})
+    for stanza in text.split("\n\n"):
+        fields: dict[str, str] = {}
+        key = None
+        for line in stanza.splitlines():
+            if line.startswith((" ", "\t")) and key:
+                fields[key] += " " + line.strip()
+            elif ":" in line:
+                key, _, value = line.partition(":")
+                fields[key] = value.strip()
+        name = fields.get("Package")
+        if not name:
+            continue
+        version = fields.get("Version", "")
+        old = index.get(name)
+        if old and debver.compare(old["bin_version"], version) >= 0:
+            continue
+        # "Source: foo (1.2-3)" when the source version differs from the binary's
+        src_field = fields.get("Source", name)
+        source = src_field.split(" ")[0]
+        m = re.search(r"\(([^)]+)\)", src_field)
+        deps = _parse_relations(fields.get("Pre-Depends", "")) + _parse_relations(fields.get("Depends", ""))
+        index[name] = {"source": source, "version": m.group(1) if m else version,
+                       "bin_version": version, "deps": deps}
+        for virt in _parse_relations(fields.get("Provides", "")):
+            lst = provides.setdefault(virt[0], [])
+            if name not in lst:
+                lst.append(name)
+
+
+def package_index(series: str, arch: str, refresh: bool = False) -> dict:
+    def build() -> dict:
+        index: dict = {}
+        for pocket in POCKETS:
+            for comp in COMPONENTS:
+                url = f"{_mirror(arch)}/dists/{series}{pocket}/{comp}/binary-{arch}/Packages.xz"
+                try:
+                    data = lzma.decompress(fetch(url))
+                except Exception:
+                    continue  # e.g. an empty pocket/component
+                parse_packages(data.decode("utf-8", "replace"), index)
+        if len(index) < 100:
+            raise RuntimeError(f"could not download the {series}/{arch} package index")
+        return index
+    return cached_json(f"packages-{series}-{arch}", build, refresh=refresh)
+
+
+def base_packages(base: str) -> set[str] | None:
+    """Packages shipped in the base snap, from its dpkg.list (None if the base isn't installed)."""
+    path = Path(f"/snap/{base}/current/usr/share/snappy/dpkg.list")
+    if not path.exists():
+        return None
+    pkgs = set()
+    for line in path.read_text().splitlines():
+        if line.startswith("ii "):
+            pkgs.add(line.split()[1].split(":")[0])
+    return pkgs
+
+
+def closure(roots: list[str], index: dict, exclude: set[str]) -> tuple[set[str], list[str]]:
+    """Binary packages pulled in by ``roots``; returns (packages, names not found)."""
+    provides = index.get("__provides__", {})
+    seen: set[str] = set()
+    missing: list[str] = []
+    todo = list(roots)
+    explicit = set(roots)
+    while todo:
+        pkg = todo.pop()
+        if pkg in seen:
+            continue
+        if pkg not in index:
+            real = provides.get(pkg)
+            if not real:
+                missing.append(pkg)
+                continue
+            pkg = real[0]
+            if pkg in seen:
+                continue
+        if pkg in exclude and pkg not in explicit:
+            continue
+        seen.add(pkg)
+        for alts in index[pkg]["deps"]:
+            # already satisfied by the base or by something we're staging
+            if any(a in exclude or a in seen for a in alts):
+                continue
+            todo.append(next((a for a in alts if a in index), alts[0]))
+    return seen, missing
+
+
+def sources_of(packages: set[str], index: dict) -> dict[str, dict]:
+    """{source package: {"binaries": [...], "version": newest source version in the archive}}"""
+    out: dict[str, dict] = {}
+    for pkg in packages:
+        entry = out.setdefault(index[pkg]["source"], {"binaries": [], "version": None})
+        entry["binaries"].append(pkg)
+        v = index[pkg]["version"]
+        if entry["version"] is None or debver.compare(v, entry["version"]) > 0:
+            entry["version"] = v
+    for entry in out.values():
+        entry["binaries"].sort()
+    return out

snap_maintainer/builders.py

@@ -0,0 +1,205 @@
+"""Milestone 3a: work out how each snap gets built and uploaded (detection only).
+
+Builders, in order of preference when several apply:
+  github-workflow  a GitHub Actions workflow builds the snap (and usually uploads it)
+  launchpad        a Launchpad snap recipe builds from the repo (and usually uploads it)
+  local            nothing automated found: build with snapcraft on this machine
+  manual           set by the user: rebuilds happen outside this tool
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.request
+
+import yaml
+
+from snap_maintainer import repo as repo_mod
+
+METHODS = ("github-workflow", "launchpad", "local", "manual")
+
+LP_SNAPS_BY_PREFIX = ("https://api.launchpad.net/devel/+snaps?ws.op=findByURLPrefix"
+                      "&url_prefix={prefix}&ws.size=300")
+
+_BUILD_ACTIONS = ("snapcore/action-build", "canonical/action-build")
+_PUBLISH_ACTIONS = ("snapcore/action-publish", "canonical/action-publish")
+_BUILD_CMD = re.compile(r"\bsnapcraft(\s+(pack|snap|remote-build)\b|\s*$|\s+--(?!classic))")
+_UPLOAD_CMD = re.compile(r"\bsnapcraft\s+(upload|push)\b")
+_RELEASE_ARG = re.compile(r"--release[= ]+[\"']?([\w/,.-]+)")
+
+
+def _runs_build(run: str) -> bool:
+    """A `run:` script that builds a snap (`snapcraft`, `snapcraft pack`, ...), ignoring
+    lines that merely install snapcraft."""
+    return any(_BUILD_CMD.search(line) for line in run.splitlines()
+               if "install" not in line and not line.lstrip().startswith("#"))
+
+
+def _triggers(doc: dict) -> dict:
+    """Normalise a workflow's `on:` (which YAML 1.1 parses as the key True)."""
+    on = doc.get("on", doc.get(True))
+    if isinstance(on, str):
+        on = {on: None}
+    elif isinstance(on, list):
+        on = {k: None for k in on}
+    elif not isinstance(on, dict):
+        on = {}
+    out = {"dispatch": "workflow_dispatch" in on, "tags": [], "branches": [],
+           "release": "release" in on, "schedule": "schedule" in on, "push": "push" in on}
+    push = on.get("push") or {}
+    if isinstance(push, dict):
+        out["tags"] = [str(t) for t in push.get("tags") or []]
+        out["branches"] = [str(b) for b in push.get("branches") or []]
+    return out
+
+
+def analyze_workflow(path: str, text: str) -> dict | None:
+    """What a workflow does with snaps, or None if it doesn't build one."""
+    try:
+        doc = yaml.safe_load(text) or {}
+    except yaml.YAMLError:
+        return None
+    if not isinstance(doc, dict):
+        return None
+    builds = uploads = False
+    how_build = how_upload = None
+    channels: set[str] = set()
+    for job in (doc.get("jobs") or {}).values():
+        for step in (job or {}).get("steps") or []:
+            if not isinstance(step, dict):
+                continue
+            uses = str(step.get("uses") or "")
+            run = str(step.get("run") or "")
+            if uses.startswith(_BUILD_ACTIONS):
+                builds, how_build = True, uses.split("@")[0]
+            elif _runs_build(run):
+                builds, how_build = True, how_build or "snapcraft"
+            if uses.startswith(_PUBLISH_ACTIONS):
+                uploads, how_upload = True, uses.split("@")[0]
+                rel = str((step.get("with") or {}).get("release") or "")
+                if rel and "$" not in rel:
+                    channels.add(rel)
+            elif _UPLOAD_CMD.search(run):
+                uploads, how_upload = True, how_upload or "snapcraft upload"
+                for m in _RELEASE_ARG.finditer(run):
+                    if "$" not in m.group(1):
+                        channels.add(m.group(1))
+    if not builds:
+        return None
+    return {
+        "path": path,
+        "name": str(doc.get("name") or path.rsplit("/", 1)[-1]),
+        "builds_with": how_build,
+        "uploads": uploads,
+        "uploads_with": how_upload,
+        "channels": sorted(channels),  # empty: none, or decided at run time
+        "triggers": _triggers(doc),
+    }
+
+
+def describe_trigger(t: dict) -> str:
+    """Short human description of how a workflow gets started."""
+    parts = []
+    if t.get("tags"):
+        parts.append("tag " + ", ".join(t["tags"]))
+    if t.get("release"):
+        parts.append("GitHub release")
+    if t.get("dispatch"):
+        parts.append("manual dispatch")
+    if t.get("branches"):
+        parts.append("push to " + ", ".join(t["branches"]))
+    elif t.get("push") and not t.get("tags"):
+        parts.append("any push")
+    if t.get("schedule"):
+        parts.append("schedule")
+    return " or ".join(parts) or "unknown trigger"
+
+
+def workflow_candidates(repo: dict) -> list[dict]:
+    out = []
+    for path in repo.get("workflows") or []:
+        code, text = repo_mod.run_gh([
+            "api", "-H", "Accept: application/vnd.github.raw",
+            f"repos/{repo['owner']}/{repo['name']}/contents/{path}"
+            f"?ref={repo.get('default_branch') or 'HEAD'}",
+        ])
+        if code == 0:
+            wf = analyze_workflow(path, text)
+            if wf:
+                out.append(wf)
+    return out
+
+
+def launchpad_recipes(owner: str) -> list[dict] | None:
+    """Launchpad snap recipes building from github.com/<owner>/*, or None if LP is unreachable."""
+    url = LP_SNAPS_BY_PREFIX.format(prefix=f"https://github.com/{owner}/")
+    try:
+        with urllib.request.urlopen(url, timeout=30) as resp:
+            data = json.load(resp)
+    except Exception:
+        return None
+    return [
+        {
+            "name": e.get("name"),
+            "owner": (e.get("owner_link") or "").rsplit("~", 1)[-1] or None,
+            "url": e.get("web_link"),
+            "git_url": e.get("git_repository_url"),
+            "git_path": e.get("git_path"),
+            "store_name": e.get("store_name"),
+            "store_upload": bool(e.get("store_upload")),
+            "auto_build": bool(e.get("auto_build")),
+            "channels": e.get("store_channels") or [],
+        }
+        for e in data.get("entries", [])
+    ]
+
+
+def _same_repo(a: str | None, b: str) -> bool:
+    return bool(a) and repo_mod.normalize(a).lower() == repo_mod.normalize(b).lower()
+
+
+def detect(snap_name: str, repo: dict, lp: list[dict] | None) -> dict:
+    """Pick a builder for one snap and record the evidence and the alternatives."""
+    candidates: list[dict] = []
+    notes: list[str] = []
+
+    for wf in workflow_candidates(repo) if repo.get("verified") else []:
+        what = "builds and uploads" if wf["uploads"] else "builds (no upload)"
+        candidates.append({
+            "method": "github-workflow",
+            "rank": 0 if wf["uploads"] else 2,
+            "workflow": wf,
+            "evidence": f"{wf['path']} {what}; runs on {describe_trigger(wf['triggers'])}",
+        })
+
+    if lp is None:
+        notes.append("Launchpad couldn't be reached; Launchpad recipes weren't checked")
+    for r in lp or []:
+        if not (_same_repo(r["git_url"], repo.get("url", "")) or
+                (repo.get("redirected_from") and _same_repo(r["git_url"], repo["redirected_from"]))):
+            continue
+        if r["store_name"] and r["store_name"] != snap_name:
+            continue
+        what = "builds and uploads" if r["store_upload"] else "builds (no upload)"
+        auto = ", automatically on new commits" if r["auto_build"] else ""
+        candidates.append({
+            "method": "launchpad",
+            "rank": 1 if r["store_upload"] else 3,
+            "recipe": r,
+            "evidence": f"Launchpad recipe {r['name']} {what}{auto}",
+        })
+
+    candidates.sort(key=lambda c: c["rank"])
+    if candidates:
+        best = candidates[0]
+        chosen = {k: v for k, v in best.items() if k != "rank"}
+    else:
+        chosen = {"method": "local",
+                  "evidence": "no workflow or Launchpad recipe builds this snap"}
+        if not repo.get("snapcraft_yaml"):
+            notes.append("no snapcraft.yaml found, so a local build isn't possible yet")
+    chosen["source"] = "detected"
+    chosen["alternatives"] = [c["evidence"] for c in candidates[1:]]
+    chosen["notes"] = notes
+    return chosen

snap_maintainer/cache.py

@@ -0,0 +1,38 @@
+"""Small on-disk cache for slow-to-fetch, slow-to-parse upstream data."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.request
+from pathlib import Path
+from typing import Callable
+
+DEFAULT_TTL = 24 * 3600
+
+
+def cache_dir() -> Path:
+    base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache")
+    d = base / "snap-maintainer"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def fetch(url: str, timeout: int = 120) -> bytes:
+    req = urllib.request.Request(url, headers={"User-Agent": "snap-maintainer"})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return resp.read()
+
+
+def cached_json(key: str, build: Callable[[], object], ttl: int = DEFAULT_TTL,
+                refresh: bool = False) -> object:
+    """Return the cached JSON value for ``key``, rebuilding it when stale."""
+    path = cache_dir() / f"{key}.json"
+    if not refresh and path.exists() and time.time() - path.stat().st_mtime < ttl:
+        return json.loads(path.read_text())
+    value = build()
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(value))
+    tmp.replace(path)
+    return value