dkview 3.0.0b5__py3-none-any.whl

dkview/features/clean.py

@@ -0,0 +1,150 @@
+"""`dkview clean`: remove old, unused image tags, keeping the newest few."""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from typing import List, Sequence, Tuple
+
+from ..ansi import BOLD, DIM, GREEN, RED, YELLOW, paint
+from ..layout import render
+from ..options import Options
+from ..runner import capture
+from ..table import Table
+from ..transform import short_image
+from ..units import ago, human_size
+from .images import (IN_USE, NOT_IN_USE, ImageInfo, by_repository, global_options,
+                     load_images)
+
+DELETE = "delete"
+
+
+@dataclass
+class Decision:
+    image: ImageInfo
+    action: str  # "delete", "keep: in use", "keep: newest"
+
+
+def plan(images: List[ImageInfo], keep: int) -> List[Decision]:
+    """Decide per tag. Per repository the `keep` newest tags always stay,
+    images used by any container (running or stopped) always stay, and
+    the rest is deleted. Dangling images (<none>) are deleted when unused."""
+    decisions = []
+    for repo, items in by_repository(images).items():
+        for rank, image in enumerate(items):
+            if image.in_use:
+                action = "keep: in use"
+            elif image.dangling:
+                action = DELETE
+            elif rank < keep:
+                action = "keep: newest"
+            else:
+                action = DELETE
+            decisions.append(Decision(image, action))
+    return decisions
+
+
+def freed_space(decisions: List[Decision]) -> float:
+    """Space released: an image ID only counts when every tag of it goes."""
+    sizes = {}
+    kept_ids = set()
+    for d in decisions:
+        if d.action == DELETE:
+            sizes[d.image.id] = d.image.size
+        else:
+            kept_ids.add(d.image.id)
+    return sum(size for image_id, size in sizes.items() if image_id not in kept_ids)
+
+
+def _style(header: str, value: str):
+    if header == "ACTION":
+        if value == DELETE:
+            return RED
+        if value.endswith("in use"):
+            return GREEN
+        return DIM
+    return None
+
+
+def plan_table(decisions: List[Decision], short: bool, show_kept: bool) -> Table:
+    table = Table(["REPOSITORY", "TAG", "IMAGE ID", "CREATED", "SIZE", "ACTION"])
+    touched = {d.image.repository for d in decisions if d.action == DELETE}
+    for d in decisions:
+        if d.image.repository not in touched:
+            continue
+        if d.action != DELETE and not show_kept:
+            continue
+        repo = d.image.repository
+        name = short_image(repo) if short else repo
+        mark = IN_USE if d.image.in_use else NOT_IN_USE
+        table.rows.append([f"{mark} {name}", d.image.tag, d.image.id,
+                           ago(d.image.created), human_size(d.image.size), d.action])
+    return table
+
+
+def confirm(question: str) -> bool:
+    if not sys.stdin.isatty():
+        sys.stderr.write("dkview: not asking without a terminal; use --yes to delete "
+                         "or --dry-run to only look.\n")
+        return False
+    try:
+        answer = input(question)
+    except EOFError:
+        return False
+    return answer.strip().lower() in ("y", "yes")
+
+
+def remove(argv: Sequence[str], targets: List[ImageInfo]) -> Tuple[int, List[str]]:
+    """Delete one reference at a time so one failure doesn't stop the rest."""
+    base = global_options(argv)
+    removed, errors = 0, []
+    for image in targets:
+        result = capture(base + ["rmi", image.reference])
+        if result.code == 0:
+            removed += 1
+            sys.stdout.write(paint("  ✓ ", GREEN) + f"removed {image.reference}\n")
+        else:
+            message = " ".join(result.stderr.split()) or f"exit code {result.code}"
+            errors.append(f"{image.reference}: {message}")
+            sys.stdout.write(paint("  ✗ ", RED) + f"{image.reference}: {message}\n")
+        sys.stdout.flush()
+    return removed, errors
+
+
+def run(argv: Sequence[str], opts: Options) -> int:
+    images = load_images(argv)
+    if images is None:
+        return 1
+    if opts.grep:
+        images = [i for i in images if opts.grep.search(f"{i.repository}:{i.tag}")]
+
+    decisions = plan(images, opts.keep)
+    targets = [d.image for d in decisions if d.action == DELETE]
+    if not targets:
+        sys.stdout.write(paint("✓ Nothing to clean: ", GREEN)
+                         + f"no unused tags beyond the newest {opts.keep} of each "
+                         "repository, and no dangling images.\n")
+        return 0
+
+    table = plan_table(decisions, opts.short, show_kept=opts.full or opts.dry_run)
+    sys.stdout.write(render(table, opts.width, _style) + "\n")
+
+    freed = freed_space(decisions)
+    repos = len({t.repository for t in targets})
+    sys.stdout.write(
+        paint(f"{len(targets)} images to delete", BOLD, RED) + f" in {repos} repositories, "
+        + paint(f"up to {human_size(freed)} freed", BOLD)
+        + paint(f"  (keeping the newest {opts.keep} per repository and every image "
+                "a container uses)", DIM) + "\n")
+
+    if opts.dry_run:
+        sys.stdout.write(paint("Dry run: nothing was deleted.", YELLOW) + "\n")
+        return 0
+    if not opts.yes and not confirm(f"Delete these {len(targets)} images? [y/N] "):
+        sys.stdout.write("Nothing was deleted.\n")
+        return 1
+
+    removed, errors = remove(argv, targets)
+    summary = f"Removed {removed} of {len(targets)} images."
+    sys.stdout.write("\n" + (paint(summary, GREEN) if not errors else paint(summary, YELLOW)) + "\n")
+    return 0 if not errors else 1

dkview/features/dashboard.py

@@ -0,0 +1,168 @@
+"""`dkview dash`: containers, services, problems and disk use on one screen."""
+
+from __future__ import annotations
+
+import json
+import re
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, List, Optional
+
+from ..ansi import BOLD, DIM, GREEN, RED, YELLOW, paint, terminal_width
+from .. import engine
+from ..formats import fields_template
+from ..layout import render
+from ..options import Options
+from ..runner import capture
+from ..styles import container_status, replicas, styler
+from ..table import Table
+from ..transform import compact_age, is_task_container, short_image, short_task_name
+
+
+def _json_lines(argv: List[str]) -> Optional[List[Dict[str, Any]]]:
+    result = capture(argv)
+    if result.code != 0:
+        return None
+    items = []
+    for line in result.stdout.splitlines():
+        line = line.strip()
+        if line:
+            try:
+                items.append(json.loads(line))
+            except ValueError:
+                pass
+    return items
+
+
+def collect() -> Dict[str, Any]:
+    """Ask docker for everything at once; `stats` alone takes ~2 seconds."""
+    program = engine.name()
+    commands = {
+        "info": [program, "info", "--format", "{{json .}}"],
+        "ps": [program, "ps", "-a", "--format",
+               fields_template(["ID", "Names", "Image", "State", "Status", "Ports"])],
+        "stats": [program, "stats", "--no-stream", "--format",
+                  fields_template(["ID", "CPUPerc", "MemUsage"])],
+        "df": [program, "system", "df", "--format", "{{json .}}"],
+    }
+    if not engine.is_podman([program]):  # podman has no swarm
+        commands["services"] = [program, "service", "ls", "--format", "{{json .}}"]
+    with ThreadPoolExecutor(len(commands)) as pool:
+        futures = {k: pool.submit(_json_lines, v) for k, v in commands.items()}
+        data = {k: f.result() for k, f in futures.items()}
+    if data["ps"] is None:  # docker too old for .State in a template
+        data["ps"] = _json_lines([program, "ps", "-a", "--format", "{{json .}}"])
+    data.setdefault("services", None)
+    return data
+
+
+def _header(info: Dict[str, Any]) -> str:
+    if "host" in info and "version" in info:  # podman's layout
+        host = info.get("host") or {}
+        info = {"ServerVersion": (info.get("version") or {}).get("Version", "?"),
+                "Name": host.get("hostname", "?"), "NCPU": host.get("cpus", "?"),
+                "MemTotal": host.get("memTotal") or 0, "Engine": "Podman"}
+    swarm = (info.get("Swarm") or {}).get("LocalNodeState", "")
+    mem = info.get("MemTotal") or 0
+    parts = [
+        f"{info.get('Engine', 'Docker')} {info.get('ServerVersion', '?')} on {info.get('Name', '?')}",
+        f"{info.get('NCPU', '?')} CPUs",
+        f"{mem / 1024 ** 3:.1f}GiB RAM" if mem else "",
+        f"swarm {swarm}" if swarm and swarm != "inactive" else "",
+    ]
+    return paint(" · ".join(p for p in parts if p), BOLD)
+
+
+def _counts(ps: List[Dict[str, Any]]) -> str:
+    running = sum(1 for c in ps if c.get("State") == "running")
+    unhealthy = sum(1 for c in ps if "unhealthy" in c.get("Status", ""))
+    failed = sum(1 for c in ps if c.get("State") == "exited"
+                 and not c.get("Status", "").startswith("Exited (0)"))
+    stopped = sum(1 for c in ps if c.get("State") != "running") - failed
+    pieces = [paint(f"{running} running", GREEN)]
+    if unhealthy:
+        pieces.append(paint(f"{unhealthy} unhealthy", RED))
+    if failed:
+        pieces.append(paint(f"{failed} failed", RED))
+    if stopped:
+        pieces.append(paint(f"{stopped} stopped", DIM))
+    return "Containers: " + " · ".join(pieces)
+
+
+def _problems(ps: List[Dict[str, Any]], services: List[Dict[str, Any]]) -> List[str]:
+    problems = []
+    for c in ps:
+        status = c.get("Status", "")
+        if container_status(status) == RED:
+            problems.append(f"container {c.get('Names')}: {compact_age(status)}")
+        elif c.get("State") == "restarting":
+            problems.append(f"container {c.get('Names')}: restarting")
+    for s in services:
+        if replicas(s.get("Replicas", "")) in (RED, YELLOW):
+            problems.append(f"service {s.get('Name')}: {s.get('Replicas')} replicas running")
+    return problems
+
+
+def frame(opts: Options, width: Optional[int] = None) -> str:
+    width = width or opts.width or terminal_width()
+    data = collect()
+    if data["ps"] is None:
+        return paint(f"Cannot reach {engine.name()}. Is it running?", RED) + "\n"
+
+    info = (data["info"] or [{}])[0]
+    stats = {s.get("ID", "")[:12]: s for s in data["stats"] or []}
+    services = data["services"] or []
+    # Swarm keeps the last few dead tasks of every service. Their health is the
+    # service's REPLICAS, so they would only repeat it, many times over.
+    ps = [c for c in data["ps"] if c.get("State") == "running"
+          or not is_task_container(c.get("Names", ""))]
+    old_tasks = len(data["ps"]) - len(ps)
+
+    out = [_header(info), _counts(ps), ""]
+
+    problems = _problems(ps, services)
+    if problems:
+        out.append(paint(f"Needs attention ({len(problems)})", BOLD, RED))
+        out += ["  " + paint("✗ ", RED) + p for p in problems]
+    else:
+        out.append(paint("✓ Everything is running and healthy", GREEN))
+    out.append("")
+
+    def order(c: Dict[str, Any]) -> tuple:
+        bad = container_status(c.get("Status", "")) == RED
+        return (not bad, c.get("State") != "running", c.get("Names", ""))
+
+    containers = Table(["NAME", "STATUS", "IMAGE", "CPU %", "MEM", "PORTS"])
+    for c in sorted(ps, key=order):
+        s = stats.get(c.get("ID", "")[:12], {})
+        mem = s.get("MemUsage", "").split(" / ")[0]
+        ports = re.sub(r"(0\.0\.0\.0|\[::\]|:::)", "", c.get("Ports", ""))
+        ports = ", ".join(dict.fromkeys(p.strip() for p in ports.split(",") if p.strip()))
+        image = c.get("Image", "")
+        containers.rows.append([
+            short_task_name(c.get("Names", "")), compact_age(c.get("Status", "")),
+            short_image(image) if opts.short else image,
+            s.get("CPUPerc", ""), mem, ports,
+        ])
+    out.append(paint("Containers", BOLD))
+    out.append(render(containers, width, styler))
+    if old_tasks:
+        out.append(paint(f"{old_tasks} stopped swarm task container{'s' if old_tasks != 1 else ''}"
+                         " not shown; `dkview doctor` explains failed tasks", DIM))
+
+    if services:
+        table = Table(["NAME", "MODE", "REPLICAS", "IMAGE", "PORTS"])
+        for s in sorted(services, key=lambda s: s.get("Name", "")):
+            image = re.sub(r"@sha256:[0-9a-f]+", "", s.get("Image", ""))
+            table.rows.append([s.get("Name", ""), s.get("Mode", ""), s.get("Replicas", ""),
+                               short_image(image) if opts.short else image, s.get("Ports", "")])
+        out += ["", paint("Services", BOLD), render(table, width, styler)]
+
+    if data["df"]:
+        table = Table(["TYPE", "TOTAL", "ACTIVE", "SIZE", "RECLAIMABLE"])
+        for d in data["df"]:
+            table.rows.append([d.get("Type", ""), str(d.get("TotalCount", d.get("Total", ""))),
+                               str(d.get("Active", "")), d.get("Size", ""),
+                               d.get("Reclaimable", "")])
+        out += ["", paint("Disk", BOLD), render(table, width, styler)]
+
+    return "\n".join(out) + "\n"

dkview/features/doctor.py

@@ -0,0 +1,259 @@
+"""`dkview doctor`: what is wrong with each swarm service, and why."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Sequence
+
+from .. import engine
+from ..ansi import BOLD, DIM, GREEN, RED, YELLOW, paint, terminal_width
+from ..layout import render
+from ..options import Options
+from ..runner import capture
+from ..styles import styler
+from ..table import Table
+from ..transform import compact_age, short_image, sort_key
+from .images import global_options
+
+OK, WARN, BAD = "ok", "warn", "bad"
+FAILED_STATES = ("failed", "rejected", "orphaned")
+
+
+@dataclass
+class Task:
+    slot: str
+    node: str
+    desired: str
+    state: str      # "Failed"
+    when: str       # "4s ago"
+    error: str
+
+    @classmethod
+    def from_json(cls, item: dict) -> "Task":
+        current = item.get("CurrentState", "")
+        state, _, when = current.partition(" ")
+        return cls(
+            slot=item.get("Name", ""),
+            node=item.get("Node", ""),
+            desired=item.get("DesiredState", ""),
+            state=state,
+            when=compact_age(when),
+            error=item.get("Error", "").strip().strip('"').strip(),
+        )
+
+    @property
+    def failed(self) -> bool:
+        return self.state.lower() in FAILED_STATES
+
+
+@dataclass
+class ServiceReport:
+    name: str
+    replicas: str
+    image: str
+    tasks: List[Task] = field(default_factory=list)
+    update_state: str = ""
+    update_message: str = ""
+
+    @property
+    def running(self) -> int:
+        m = re.match(r"(\d+)/(\d+)", self.replicas)
+        return int(m.group(1)) if m else 0
+
+    @property
+    def wanted(self) -> int:
+        m = re.match(r"(\d+)/(\d+)", self.replicas)
+        return int(m.group(2)) if m else 0
+
+    @property
+    def failures(self) -> List[Task]:
+        return [t for t in self.tasks if t.failed]
+
+    @property
+    def health(self) -> str:
+        if self.running < self.wanted:
+            return BAD
+        if self.failures or self.update_state in ("paused", "rollback_started",
+                                                  "rollback_paused"):
+            return WARN
+        return OK
+
+
+def _json_lines(text: str) -> List[dict]:
+    items = []
+    for line in text.splitlines():
+        try:
+            items.append(json.loads(line))
+        except ValueError:
+            pass
+    return items
+
+
+def collect(argv: Sequence[str]) -> Optional[List[ServiceReport]]:
+    base = global_options(argv)
+    listing = capture(base + ["service", "ls", "--format", "{{json .}}"])
+    if listing.code != 0:
+        message = listing.stderr.strip()
+        if "not a swarm manager" in message or "swarm" in message.lower():
+            sys.stderr.write("dkview doctor: this node is not a swarm manager, "
+                             "so there are no services to check.\n")
+        else:
+            sys.stderr.write(listing.stderr)
+        return None
+
+    services = [ServiceReport(s.get("Name", ""), s.get("Replicas", ""),
+                              re.sub(r"@sha256:[0-9a-f]+", "", s.get("Image", "")))
+                for s in _json_lines(listing.stdout)]
+    if not services:
+        return []
+
+    def tasks_of(name: str) -> List[Task]:
+        result = capture(base + ["service", "ps", name, "--no-trunc", "--format", "{{json .}}"])
+        return [Task.from_json(t) for t in _json_lines(result.stdout)]
+
+    with ThreadPoolExecutor(min(8, len(services))) as pool:
+        for service, tasks in zip(services, pool.map(tasks_of, [s.name for s in services])):
+            service.tasks = tasks
+
+    inspect = capture(base + ["service", "inspect", "--format", "{{json .}}"]
+                      + [s.name for s in services])
+    for service, data in zip(services, _json_lines(inspect.stdout)):
+        status = data.get("UpdateStatus") or {}
+        service.update_state = status.get("State", "") or ""
+        service.update_message = status.get("Message", "") or ""
+    return services
+
+
+# ------------------------------------------------------------------ hints
+
+_HINTS = [
+    (r"failed to resolve reference|no such image|manifest unknown|not found: manifest|"
+     r"pull access denied|unauthorized",
+     "The image can't be pulled. Check the image name and tag, and that the "
+     "node can log in to the registry (deploy with --with-registry-auth)."),
+    (r"non-zero exit \(137\)",
+     "The container was killed (exit 137): usually out of memory or a failed "
+     "health check. Check memory limits and `dkview errors {name}`."),
+    (r"non-zero exit \((\d+)\)",
+     "The program inside the container exited with an error. "
+     "See why with `dkview errors {name}`."),
+    (r"no suitable node",
+     "No node matches the service's placement constraints or resources. "
+     "Check constraints, labels and reserved CPU/memory."),
+    (r"port .* is already in use|address already in use",
+     "A published port is already taken on the node."),
+    (r"invalid mount|bind source path does not exist|no such file or directory",
+     "A volume or bind mount source doesn't exist on the node."),
+    (r"unhealthy container|health check",
+     "The container's health check keeps failing. "
+     "Check `dkview inspect` on a task container and the service logs."),
+]
+
+
+def hint(error: str, service: str) -> str:
+    for pattern, text in _HINTS:
+        if re.search(pattern, error, re.IGNORECASE):
+            return text.format(name=service)
+    return ""
+
+
+# -------------------------------------------------------------- rendering
+
+def error_table(report: ServiceReport, every_task: bool) -> Table:
+    if every_task:
+        table = Table(["TASK", "NODE", "STATE", "WHEN", "ERROR"])
+        for t in report.failures:
+            table.rows.append([t.slot, t.node, t.state, t.when, t.error])
+        return table
+
+    groups: "OrderedDict[str, List[Task]]" = OrderedDict()
+    for t in report.failures:
+        groups.setdefault(t.error or "(no error message)", []).append(t)
+    table = Table(["TIMES", "STATE", "LAST", "NODES", "ERROR"])
+    for error, tasks in groups.items():
+        latest = min(tasks, key=lambda t: sort_key(t.when))
+        states = ", ".join(OrderedDict.fromkeys(t.state for t in tasks))
+        nodes = ", ".join(OrderedDict.fromkeys(t.node for t in tasks if t.node))
+        table.rows.append([f"{len(tasks)}×", states, latest.when, nodes, error])
+    return table
+
+
+def _current(report: ServiceReport) -> str:
+    latest: Dict[str, Task] = OrderedDict()
+    for t in report.tasks:  # docker lists the newest task of each slot first
+        latest.setdefault(t.slot, t)
+    states = [f"{t.state} {t.when}".strip() for t in latest.values()
+              if t.desired.lower() != "shutdown" or t.failed]
+    return "; ".join(OrderedDict.fromkeys(states))
+
+
+def render_report(services: List[ServiceReport], opts: Options) -> str:
+    out: List[str] = []
+    counts = {h: sum(1 for s in services if s.health == h) for h in (OK, WARN, BAD)}
+    pieces = [paint(f"{counts[OK]} healthy", GREEN)]
+    if counts[WARN]:
+        pieces.append(paint(f"{counts[WARN]} restarting", YELLOW))
+    if counts[BAD]:
+        pieces.append(paint(f"{counts[BAD]} failing", RED))
+    out.append(paint("Swarm services: ", BOLD) + " · ".join(pieces))
+    out.append("")
+
+    order = {BAD: 0, WARN: 1, OK: 2}
+    problems = sorted((s for s in services if s.health != OK),
+                      key=lambda s: (order[s.health], s.name))
+    for s in problems:
+        icon, color = ("✗", RED) if s.health == BAD else ("!", YELLOW)
+        image = short_image(s.image) if opts.short else s.image
+        out.append(paint(f"{icon} {s.name}", BOLD, color) + "  "
+                   + paint(f"{s.replicas} replicas", color) + paint(f" · {image}", DIM))
+        restarts = len(s.failures)
+        details = [f"{restarts} failed tasks in recent history" if restarts else ""]
+        current = _current(s)
+        if current:
+            details.append(f"now: {current}")
+        if s.update_state:
+            details.append(f"update {s.update_state}"
+                           + (f": {s.update_message}" if s.update_message else ""))
+        out.append("  " + " · ".join(d for d in details if d))
+        if s.failures:
+            table = error_table(s, opts.full)
+            out += ["  " + line for line in render(table, (opts.width or terminal_width()) - 2, styler).splitlines()]
+            seen = set()
+            for row in table.rows:
+                text = hint(row[-1], s.name)
+                if text and text not in seen:
+                    seen.add(text)
+                    out.append("  " + paint("→ ", YELLOW) + text)
+        out.append("")
+
+    healthy = [s for s in services if s.health == OK]
+    if healthy:
+        table = Table(["SERVICE", "REPLICAS", "IMAGE"])
+        for s in sorted(healthy, key=lambda s: s.name):
+            table.rows.append([s.name, s.replicas, short_image(s.image) if opts.short else s.image])
+        out.append(paint("✓ Healthy", BOLD, GREEN))
+        out.append(render(table, opts.width or terminal_width(), styler))
+    elif not problems:
+        out.append("No services.")
+    return "\n".join(out).rstrip("\n") + "\n"
+
+
+def run(argv: Sequence[str], opts: Options) -> int:
+    if engine.is_podman(argv):
+        sys.stdout.write("Podman has no swarm services, so there is nothing for doctor "
+                         "to check. `dkview errors` shows problems in container logs.\n")
+        return 0
+    services = collect(argv)
+    if services is None:
+        return 1
+    if opts.grep:
+        services = [s for s in services if opts.grep.search(s.name)]
+    sys.stdout.write(render_report(services, opts))
+    if opts.full is False and any(s.failures for s in services):
+        sys.stdout.write(paint("(identical errors are grouped; --full lists every task)\n", DIM))
+    return 1 if any(s.health == BAD for s in services) else 0