whyfull 0.1.0__py3-none-any.whl

whyfull/__init__.py

@@ -0,0 +1,3 @@
+"""whyfull: find out why a Linux disk is full, and how to safely free space."""
+
+__version__ = "0.1.0"

whyfull/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from whyfull.cli import main
+
+sys.exit(main())

whyfull/analyze.py

@@ -0,0 +1,167 @@
+"""Run the checks against the chosen filesystems and assemble a report."""
+
+import os
+
+from whyfull import __version__
+from whyfull.checks import Context, load
+from whyfull.model import CAUTION, INFO, REVIEW, SAFE, Finding
+from whyfull.mounts import containing, filesystems
+from whyfull.units import human
+from whyfull.walk import Budget, scan
+
+DEFAULT_THRESHOLD = 85.0
+
+
+class Target(object):
+    """One filesystem under analysis and everything found about it."""
+
+    def __init__(self, fs):
+        self.fs = fs
+        self.findings = []
+        self.tree = None
+        self.largest = []  # [(display path, bytes, [(child path, bytes)])]
+        self.crowded = []  # [(entries, display path)]
+
+    def reclaimable(self):
+        out = {SAFE: 0, REVIEW: 0, CAUTION: 0}
+        for f in self.findings:
+            if f.risk in out:
+                out[f.risk] += f.bytes
+        out["total"] = out[SAFE] + out[REVIEW] + out[CAUTION]
+        return out
+
+    def largest_dirs(self, system, top=6, children=3, min_share=0.01):
+        """[(display path, bytes, [(child display path, bytes)])] from the scan."""
+        if self.tree is None:
+            return []
+        root = system.path(self.fs.mountpoint).rstrip("/") or "/"
+        floor = self.fs.used * min_share
+
+        def kids(parent):
+            out = [(p, b) for p, b in self.tree.sizes.items()
+                   if os.path.dirname(p) == parent and p != parent and b >= floor]
+            return sorted(out, key=lambda pb: -pb[1])
+
+        result = []
+        for path, nbytes in kids(root)[:top]:
+            sub = [(system.display(p), b) for p, b in kids(path)[:children]]
+            result.append((system.display(path), nbytes, sub))
+        return result
+
+
+class Report(object):
+    def __init__(self):
+        self.version = __version__
+        self.filesystems = []
+        self.targets = []
+        self.threshold = DEFAULT_THRESHOLD
+        self.all_clear = False  # no filesystem above the threshold
+        self.notes = []
+        self.errors = []  # (check id, message)
+
+
+def choose_targets(system, all_fs, paths, threshold):
+    if paths:
+        chosen = []
+        for p in paths:
+            fs = containing(system, os.path.abspath(p) if system.root == "/" else p)
+            if fs is None:
+                raise ValueError("can't find the filesystem for %s" % p)
+            if fs.dev not in [t.dev for t in chosen]:
+                chosen.append(fs)
+        return chosen, False
+    full = [fs for fs in all_fs if fs.percent >= threshold]
+    if full:
+        return sorted(full, key=lambda f: -f.percent), False
+    root = containing(system, "/", all_fs) or containing(system, "/")
+    return ([root] if root else []), True
+
+
+def analyze(system, paths=None, threshold=DEFAULT_THRESHOLD, scan_seconds=30, check_seconds=60,
+            only=None, skip=None, do_scan=True):
+    report = Report()
+    report.threshold = threshold
+    all_fs = filesystems(system)
+    report.filesystems = all_fs
+    targets, report.all_clear = choose_targets(system, all_fs, paths, threshold)
+    report.targets = [Target(fs) for fs in targets]
+    if not targets:
+        return report
+
+    known = {fs.dev: fs for fs in all_fs}
+    for fs in targets:
+        known.setdefault(fs.dev, fs)
+    ctx = Context(system, targets, Budget(check_seconds), list(known.values()))
+
+    for chk in load():
+        if only and chk.id not in only:
+            continue
+        if skip and chk.id in skip:
+            continue
+        try:
+            found = list(chk.fn(ctx) or [])
+        except Exception as e:  # a broken check must never hide the others
+            report.errors.append((chk.id, "%s: %s" % (type(e).__name__, e)))
+            continue
+        by_dev = {t.fs.dev: t for t in report.targets}
+        for f in found:
+            t = by_dev.get(f.dev)
+            if t is not None and (f.bytes > 0 or f.risk == INFO):
+                t.findings.append(f)
+    if ctx.budget.hit:
+        ctx.note("Some checks hit their time limit; sizes may be understated.")
+
+    for t in report.targets:
+        if do_scan:
+            t.tree = scan(system.path(t.fs.mountpoint), t.fs.dev, Budget(scan_seconds))
+            if not t.tree.complete:
+                ctx.note("The scan of %s stopped at its time limit; raise it with --scan-time." % t.fs.mountpoint)
+            if t.tree.denied and not system.is_root:
+                ctx.note("%d directories on %s couldn't be read; run with sudo for a complete picture."
+                         % (t.tree.denied, t.fs.mountpoint))
+            t.largest = t.largest_dirs(system)
+            t.crowded = [(n, system.display(p)) for n, p in t.tree.crowded]
+            gap = unexplained(system, t, ctx.deleted_open_bytes.get(t.fs.dev, 0), all_fs)
+            if gap is not None:
+                t.findings.append(gap)
+        t.findings.sort(key=lambda f: (-f.priority, -f.bytes))
+
+    report.notes = ctx.notes
+    return report
+
+
+def unexplained(system, target, deleted_open, all_fs):
+    """An INFO finding when df's 'used' is far above what the files add up to."""
+    tree, fs = target.tree, target.fs
+    if tree is None or not tree.complete or tree.denied:
+        return None
+    gap = fs.used - tree.total - deleted_open
+    if gap < max(512 * 2 ** 20, 0.05 * fs.used):
+        return None
+    root = fs.mountpoint.rstrip("/")
+    covering = sorted(m.mountpoint for m in all_fs
+                      if m.dev != fs.dev and m.mountpoint != fs.mountpoint
+                      and (m.mountpoint + "/").startswith(root + "/"))
+    fix = []
+    if covering:
+        fix = ["sudo mkdir -p /mnt/whyfull-under && sudo mount --bind %s /mnt/whyfull-under" % fs.mountpoint]
+        fix += ["sudo du -sh /mnt/whyfull-under%s" % m[len(root):] for m in covering[:6]]
+        fix += ["sudo umount /mnt/whyfull-under"]
+    if fs.fstype == "btrfs":
+        fix += ["sudo btrfs subvolume list -s %s  # snapshots keep deleted data" % fs.mountpoint]
+    if fs.fstype == "zfs":
+        fix += ["zfs list -t snapshot -o name,used -s used"]
+    causes = []
+    if covering:
+        causes.append("files written into a directory before another disk was mounted over it "
+                      "(%s): they still use space but are hidden" % ", ".join(covering[:4]))
+    if fs.fstype in ("btrfs", "zfs"):
+        causes.append("snapshots holding on to deleted data")
+    causes.append("filesystem metadata and journal")
+    return Finding(
+        "unexplained", fs.mountpoint,
+        "%s in use on %s isn't in any file whyfull can see" % (human(gap), fs.mountpoint),
+        gap, INFO, fs.dev, priority=10,
+        detail="Likely causes: " + "; ".join(causes) + ".",
+        fix=fix,
+    )

whyfull/checks/__init__.py

@@ -0,0 +1,92 @@
+"""Checks find specific, explainable causes of disk usage.
+
+A check is a function decorated with @check. It receives a Context and
+returns (or yields) Finding objects. Checks must never modify the system:
+they read files and run read-only commands only.
+
+To add one, create a module in this package, decorate a function with
+@check("my-id", "What it looks for"), and add the module name to MODULES
+below. See CONTRIBUTING.md for the full walkthrough.
+"""
+
+import importlib
+
+from whyfull.walk import du
+
+REGISTRY = []
+
+MODULES = [
+    "deleted_open",
+    "space",
+    "journald",
+    "logs",
+    "containers",
+    "packages",
+    "kernels",
+    "snaps",
+    "crashes",
+    "tmpfiles",
+    "usercaches",
+]
+
+
+class Check(object):
+    def __init__(self, id, title, fn):
+        self.id = id
+        self.title = title
+        self.fn = fn
+
+
+def check(id, title):
+    def register(fn):
+        REGISTRY.append(Check(id, title, fn))
+        return fn
+    return register
+
+
+def load():
+    for name in MODULES:
+        importlib.import_module("whyfull.checks." + name)
+    return list(REGISTRY)
+
+
+class Context(object):
+    """What a check gets to work with.
+
+    system   -- whyfull.system.System; use it for every file and command access
+    targets  -- the Filesystem objects being analysed
+    budget   -- shared time budget for size measurements
+    """
+
+    def __init__(self, system, targets, budget, all_filesystems=None):
+        self.system = system
+        self.targets = targets
+        self.budget = budget
+        self.all_filesystems = all_filesystems or targets
+        self.target_devs = set(fs.dev for fs in targets)
+        self.notes = []
+        self.deleted_open_bytes = {}  # dev -> bytes, filled by the deleted_open check
+        self._seen = set()
+
+    def dev_of(self, path):
+        st = self.system.lstat(path)
+        return None if st is None else st.st_dev
+
+    def on_target(self, path):
+        return self.dev_of(path) in self.target_devs
+
+    def size(self, path):
+        """Allocated bytes under path (on its own filesystem), or 0 if missing."""
+        local = self.system.path(path)
+        usage = du(local, budget=self.budget)
+        return usage.bytes
+
+    def filesystem(self, dev):
+        for fs in self.all_filesystems:
+            if fs.dev == dev:
+                return fs
+        return None
+
+    def note(self, text):
+        if text not in self.notes:
+            self.notes.append(text)

whyfull/checks/containers.py

@@ -0,0 +1,94 @@
+"""Docker images, build cache, stopped containers, volumes and container logs."""
+
+import json
+import os
+
+from whyfull.checks import check
+from whyfull.model import CAUTION, REVIEW, SAFE, Finding
+from whyfull.units import human, parse_size
+
+MIN_RECLAIM = 200 * 10 ** 6
+BIG_LOG = 500 * 10 ** 6
+
+# docker system df row type -> (risk, title, fix, detail)
+DF_ROWS = {
+    "Build Cache": (
+        SAFE, "Docker build cache", ["docker builder prune -f"],
+        "Layers cached by docker build. They are rebuilt on the next build that needs them.",
+    ),
+    "Images": (
+        REVIEW, "Docker images not used by any container", ["docker image prune -a -f"],
+        "Images with no container using them. They are pulled again if something needs them later.",
+    ),
+    "Containers": (
+        REVIEW, "Stopped Docker containers", ["docker container prune -f"],
+        "Writable layers of containers that have exited. Check nothing needs to be restarted from them.",
+    ),
+    "Local Volumes": (
+        CAUTION, "Docker volumes not used by any container", ["docker volume ls -f dangling=true  # look first",
+                                                               "docker volume prune -f"],
+        "Volumes hold data such as databases. Unused ones may still matter: inspect before pruning.",
+    ),
+}
+
+
+def docker_root(ctx):
+    out = ctx.system.run(["docker", "info", "--format", "{{.DockerRootDir}}"])
+    root = (out or "").strip()
+    return root or "/var/lib/docker"
+
+
+@check("docker", "Docker images, build cache, containers and volumes")
+def docker(ctx):
+    root = docker_root(ctx)
+    if not ctx.system.exists(root) or not ctx.on_target(root):
+        return
+    dev = ctx.dev_of(root)
+    out = ctx.system.run(["docker", "system", "df", "--format", "{{json .}}"], timeout=60)
+    if out is None:
+        ctx.note("Docker data lives on this disk, but `docker system df` didn't run; "
+                 "run whyfull with sudo or as a member of the docker group to include it.")
+    else:
+        for line in out.splitlines():
+            try:
+                row = json.loads(line)
+            except ValueError:
+                continue
+            spec = DF_ROWS.get(row.get("Type"))
+            if spec is None:
+                continue
+            reclaim = parse_size(str(row.get("Reclaimable", "")).split(" (")[0])
+            if not reclaim or reclaim < MIN_RECLAIM:
+                continue
+            risk, title, fix, detail = spec
+            yield Finding("docker", row["Type"].lower().replace(" ", "-"), title, reclaim, risk, dev,
+                          detail=detail, fix=fix,
+                          evidence=["%s total, %s in use" % (row.get("TotalCount", "?"), row.get("Active", "?"))])
+
+    # Container logs are not in `docker system df`; read them from disk.
+    logs = []
+    containers = os.path.join(root, "containers")
+    for cid in ctx.system.listdir(containers):
+        log = os.path.join(containers, cid, cid + "-json.log")
+        st = ctx.system.stat(log)
+        if st is None or st.st_size < BIG_LOG:
+            continue
+        name = cid[:12]
+        cfg = ctx.system.read_text(os.path.join(containers, cid, "config.v2.json"))
+        if cfg:
+            try:
+                name = json.loads(cfg).get("Name", "").lstrip("/") or name
+            except ValueError:
+                pass
+        logs.append((st.st_blocks * 512, name, log))
+    logs.sort(reverse=True)
+    for size, name, _log in logs[:5]:
+        yield Finding(
+            "docker", "log:" + name, "Docker log of container %s is %s" % (name, human(size)),
+            size, CAUTION, dev,
+            detail=("The json-file log driver keeps everything a container prints unless "
+                    "max-size is set. Truncating loses that history; set rotation so it doesn't grow back."),
+            fix=["sudo truncate -s 0 \"$(docker inspect --format '{{.LogPath}}' %s)\"" % name,
+                 "# rotate future logs (applies to new containers): add to /etc/docker/daemon.json",
+                 '#   {"log-driver": "json-file", "log-opts": {"max-size": "50m", "max-file": "3"}}'],
+        )

whyfull/checks/crashes.py

@@ -0,0 +1,31 @@
+"""Crash reports and core dumps."""
+
+from whyfull.checks import check
+from whyfull.model import REVIEW, Finding
+
+MIN_BYTES = 100 * 10 ** 6
+
+PLACES = [
+    ("/var/lib/systemd/coredump", "systemd-coredump core dumps",
+     ["sudo find /var/lib/systemd/coredump -type f -delete",
+      "# cap them: set MaxUse=1G in /etc/systemd/coredump.conf"]),
+    ("/var/crash", "crash reports in /var/crash", ["sudo find /var/crash -type f -delete"]),
+    ("/var/lib/apport/coredump", "apport core dumps", ["sudo find /var/lib/apport/coredump -type f -delete"]),
+    ("/var/spool/abrt", "ABRT crash reports", ["sudo abrt-cli remove /var/spool/abrt/*"]),
+]
+
+
+@check("crashes", "Core dumps and crash reports")
+def run(ctx):
+    for path, title, fix in PLACES:
+        if not ctx.system.exists(path) or not ctx.on_target(path):
+            continue
+        size = ctx.size(path)
+        if size < MIN_BYTES:
+            continue
+        yield Finding(
+            "crashes", path, title, size, REVIEW, ctx.dev_of(path),
+            detail=("Memory dumps from crashed programs. Useful for debugging a recent crash; "
+                    "otherwise safe to delete."),
+            fix=fix,
+        )

whyfull/checks/deleted_open.py

@@ -0,0 +1,125 @@
+"""Files that were deleted while a process still had them open.
+
+The classic "df says full, du says empty" problem: deleting a log file that a
+service is writing to removes its name, but the space is only released when
+the last process closes it.
+"""
+
+import os
+import re
+
+from whyfull.checks import check
+from whyfull.model import CAUTION, Finding
+from whyfull.walk import allocated
+
+MIN_BYTES = 1 << 20  # ignore groups smaller than 1 MiB
+
+_UNIT_RE = re.compile(r"/([^/]+\.service)(?:/|$)")
+_DOCKER_RE = re.compile(r"docker[-/]([0-9a-f]{12,64})")
+
+
+def _read(path):
+    try:
+        with open(path) as f:
+            return f.read()
+    except OSError:
+        return ""
+
+
+def owner_of(proc, pid):
+    """('unit', 'nginx.service') / ('container', 'abc123...') / ('process', 'name')."""
+    cgroup = _read(os.path.join(proc, pid, "cgroup"))
+    m = _DOCKER_RE.search(cgroup)
+    if m:
+        return "container", m.group(1)[:12]
+    units = _UNIT_RE.findall(cgroup)
+    # Skip per-user manager units; user@1000.service is not what holds the file.
+    units = [u for u in units if not u.startswith("user@")]
+    if units:
+        return "unit", units[-1]
+    comm = _read(os.path.join(proc, pid, "comm")).strip() or "pid " + pid
+    return "process", comm
+
+
+def scan_proc(proc):
+    """Yield (pid, fd, target, stat) for every open file that has been deleted."""
+    try:
+        pids = [p for p in os.listdir(proc) if p.isdigit()]
+    except OSError:
+        return
+    for pid in pids:
+        fd_dir = os.path.join(proc, pid, "fd")
+        try:
+            fds = os.listdir(fd_dir)
+        except OSError:
+            continue
+        for fd in fds:
+            fd_path = os.path.join(fd_dir, fd)
+            try:
+                target = os.readlink(fd_path)
+            except OSError:
+                continue
+            if not target.endswith(" (deleted)") or not target.startswith("/"):
+                continue
+            if target.startswith("/memfd:"):
+                continue
+            try:
+                st = os.stat(fd_path)
+            except OSError:
+                continue
+            if st.st_nlink != 0:
+                continue
+            yield pid, fd, target[: -len(" (deleted)")], st
+
+
+@check("deleted-open", "Deleted files still held open by a process")
+def run(ctx):
+    groups = {}  # (dev, kind, name) -> dict
+    seen = set()
+    for pid, fd, path, st in scan_proc(ctx.system.proc):
+        if st.st_dev not in ctx.target_devs:
+            continue
+        kind, name = owner_of(ctx.system.proc, pid)
+        g = groups.setdefault((st.st_dev, kind, name), {"bytes": 0, "files": {}, "fds": []})
+        g["fds"].append((pid, fd))
+        key = (st.st_dev, st.st_ino)
+        if key in seen:
+            continue
+        seen.add(key)
+        size = allocated(st)
+        g["bytes"] += size
+        g["files"][path] = g["files"].get(path, 0) + size
+
+    if not ctx.system.is_root:
+        ctx.note("Run with sudo to see deleted files held open by other users' processes.")
+
+    for (dev, kind, name), g in groups.items():
+        ctx.deleted_open_bytes[dev] = ctx.deleted_open_bytes.get(dev, 0) + g["bytes"]
+        if g["bytes"] < MIN_BYTES:
+            continue
+        nfiles = len(g["files"])
+        noun = "file" if nfiles == 1 else "files"
+        if kind == "unit":
+            short = name[: -len(".service")]
+            title = "%d deleted %s still open by %s" % (nfiles, noun, name)
+            fix = ["sudo systemctl restart %s" % short]
+        elif kind == "container":
+            title = "%d deleted %s still open in container %s" % (nfiles, noun, name)
+            fix = ["docker restart %s" % name]
+        else:
+            title = "%d deleted %s still open by %s" % (nfiles, noun, name)
+            fix = ["# restart %s, or free the space without a restart:" % name]
+        # Truncating through /proc frees the space immediately and keeps the process running.
+        truncate = ["sudo truncate -s 0 /proc/%s/fd/%s" % (p, f) for p, f in sorted(set(g["fds"]))[:5]]
+        if kind != "process":
+            fix.append("# or free the space without a restart:")
+        fix.extend(truncate)
+        biggest = sorted(g["files"].items(), key=lambda kv: -kv[1])[:5]
+        evidence = ["%s (deleted)" % (ctx.system.display(p) if p.startswith(ctx.system.root + "/") else p)
+                    for p, _ in biggest]
+        yield Finding(
+            "deleted-open", "%s:%s" % (kind, name), title, g["bytes"], CAUTION, dev,
+            detail=("These files were deleted, but the process writing to them never closed "
+                    "them, so the space is still in use. du can't see it; df can."),
+            fix=fix, evidence=evidence,
+        )

whyfull/checks/journald.py

@@ -0,0 +1,33 @@
+"""The systemd journal."""
+
+from whyfull.checks import check
+from whyfull.model import REVIEW, Finding
+from whyfull.units import human
+
+KEEP = 200 * 10 ** 6  # what we suggest vacuuming down to
+MIN_RECLAIM = 100 * 10 ** 6
+
+
+@check("journald", "systemd journal")
+def run(ctx):
+    for path in ("/var/log/journal", "/run/log/journal"):
+        if not ctx.system.exists(path) or not ctx.on_target(path):
+            continue
+        size = ctx.size(path)
+        reclaim = size - KEEP
+        if reclaim < MIN_RECLAIM:
+            continue
+        yield Finding(
+            "journald", path, "systemd journal in %s is %s" % (path, human(size)),
+            reclaim, REVIEW, ctx.dev_of(path),
+            detail=("journald keeps old logs until it reaches its size cap (by default 10%% of "
+                    "the filesystem). Vacuuming keeps the newest %s; capping it stops it "
+                    "growing back." % human(KEEP)),
+            fix=[
+                "sudo journalctl --vacuum-size=200M",
+                "# keep it capped:",
+                "sudo mkdir -p /etc/systemd/journald.conf.d",
+                "printf '[Journal]\\nSystemMaxUse=200M\\n' | sudo tee /etc/systemd/journald.conf.d/size.conf",
+                "sudo systemctl restart systemd-journald",
+            ],
+        )

whyfull/checks/kernels.py

@@ -0,0 +1,82 @@
+"""Old kernels: their modules under /lib/modules and their images in /boot."""
+
+import os
+import re
+
+from whyfull.checks import check
+from whyfull.model import REVIEW, Finding
+from whyfull.units import human
+
+MIN_BYTES = 50 * 10 ** 6
+BOOT_PATTERNS = ["vmlinuz-%s", "initrd.img-%s", "initramfs-%s.img", "initramfs-%skdump.img",
+                 "System.map-%s", "config-%s", "abi-%s", "retpoline-%s"]
+
+FIXES = {
+    "debian": ["sudo apt autoremove --purge",
+               "# if old kernels remain, purge them by name, e.g.: sudo apt purge linux-image-%(v)s"],
+    "rhel": ["sudo dnf remove --oldinstallonly --setopt installonly_limit=2 kernel"],
+    "suse": ["sudo zypper purge-kernels"],
+}
+
+
+def version_key(v):
+    return [int(p) if p.isdigit() else p for p in re.split(r"(\d+)", v)]
+
+
+def installed(ctx):
+    """Kernel versions with a modules directory, and where each directory is."""
+    found = {}
+    for base in ("/lib/modules", "/usr/lib/modules"):
+        for v in ctx.system.listdir(base):
+            path = os.path.join(base, v)
+            local = ctx.system.path(path)
+            if not os.path.isdir(local):
+                continue
+            real = os.path.realpath(local)
+            if any(os.path.realpath(ctx.system.path(p)) == real for p in found.values()):
+                continue
+            found.setdefault(v, path)
+    return found
+
+
+@check("kernels", "Old kernels")
+def run(ctx):
+    kernels = installed(ctx)
+    if len(kernels) < 2:
+        return
+    running = ctx.system.kernel_release()
+    newest = max(kernels, key=version_key)
+    old = sorted((v for v in kernels if v not in (running, newest)), key=version_key)
+    if not old:
+        return
+
+    per_dev = {}  # dev -> [bytes, versions]
+    for v in old:
+        parts = [kernels[v]] + ["/boot/" + p % v for p in BOOT_PATTERNS]
+        for path in parts:
+            if not ctx.system.exists(path) or not ctx.on_target(path):
+                continue
+            entry = per_dev.setdefault(ctx.dev_of(path), [0, set()])
+            entry[0] += ctx.size(path)
+            entry[1].add(v)
+
+    family = ctx.system.distro_family()
+    for dev, (nbytes, versions) in per_dev.items():
+        if nbytes < MIN_BYTES:
+            continue
+        versions = sorted(versions, key=version_key)
+        fs = ctx.filesystem(dev)
+        where = " in /boot" if fs is not None and fs.mountpoint == "/boot" else ""
+        fix = [f % {"v": versions[0]} for f in FIXES.get(family, [])] or \
+            ["# remove the kernel packages for: " + ", ".join(versions)]
+        yield Finding(
+            "kernels", str(dev), "%d old kernel%s%s" % (len(versions), "" if len(versions) == 1 else "s", where),
+            nbytes, REVIEW, dev,
+            detail=("Kernels other than the running one (%s) and the newest (%s). Package managers keep "
+                    "them as fallbacks; one spare is plenty. A full /boot can make kernel updates fail."
+                    % (running, newest)),
+            fix=fix,
+            evidence=["%s" % v for v in versions[:8]],
+        )
+    if per_dev and not any(b >= MIN_BYTES for b, _ in per_dev.values()):
+        ctx.note("Old kernels found but they are small (%s)." % human(sum(b for b, _ in per_dev.values())))