python-corekit 0.4.1__py3-none-any.whl → 0.6.0__py3-none-any.whl

corekit/connections/sql/connection.py

@@ -330,15 +330,19 @@ class SQLConnection(Connectable):
         """
         return list(self._exec(query.build()))
 
-    def delete_many_by_query(self, query: Query) -> None:
+    def delete_many_by_query(self, query: Query) -> int:
         """
         Delete every row matching a Query.
+
+        Returns the number of rows deleted.
         """
         self.require_writable()
         self._start_session()
-        for item in self.fetch_many_by_query(query):
+        items = self.fetch_many_by_query(query)
+        for item in items:
             self.session.delete(item)
         self.session.commit()
+        return len(items)
 
     def execute(self, operation: Operation) -> Any:
         """

corekit/docker/__init__.py

@@ -1,7 +1,9 @@
 """
-Docker container control.
+Docker container control and disk inspection.
 """
 
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.disk import ContainerLog, DfCategory, DockerDiskInspector, ImageSize
 from corekit.docker.watchdog import Watchdog
 
-__all__ = ["Watchdog"]
+__all__ = ["ContainerLog", "DfCategory", "DockerClientProvider", "DockerDiskInspector", "ImageSize", "Watchdog"]

corekit/docker/client.py

@@ -0,0 +1,48 @@
+"""
+Getting a Docker SDK client without assuming where the daemon lives.
+
+``DockerClientProvider`` is the one place that resolves the endpoint:
+an explicit host wins, then ``$DOCKER_HOST``, then the first socket that
+actually exists among the usual locations (including snap's), and only
+then the classic default. ``Watchdog`` and ``DockerDiskInspector`` both
+build their clients through here, so a daemon that moves (classic install
+to snap, say) is found by both.
+"""
+
+from __future__ import annotations
+
+import os
+
+from docker import DockerClient
+
+from corekit.docker.constants import DEFAULT_DOCKER_HOST, DOCKER_SOCKET_CANDIDATES
+
+__all__ = ["DockerClientProvider"]
+
+
+class DockerClientProvider:
+    """
+    Resolves Docker endpoints and builds SDK clients for them.
+    """
+
+    @staticmethod
+    def resolve_host(docker_host: str | None = None) -> str:
+        """
+        Resolve the endpoint to use, without connecting to it.
+        """
+        if docker_host:
+            return docker_host
+        from_env = os.environ.get("DOCKER_HOST")
+        if from_env:
+            return from_env
+        for candidate in DOCKER_SOCKET_CANDIDATES:
+            if candidate.exists():
+                return f"unix://{candidate}"
+        return DEFAULT_DOCKER_HOST
+
+    @staticmethod
+    def create_client(docker_host: str | None = None) -> DockerClient:
+        """
+        Build an SDK client for the resolved endpoint.
+        """
+        return DockerClient(base_url=DockerClientProvider.resolve_host(docker_host))

corekit/docker/constants.py

@@ -0,0 +1,19 @@
+"""
+Shared Docker daemon constants.
+"""
+
+from pathlib import Path
+
+DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
+
+# Candidate sockets in preference order. Snap Docker does not use
+# /var/run/docker.sock, which is the single most common reason an SDK
+# client "cannot connect" on an Ubuntu host.
+DOCKER_SOCKET_CANDIDATES: tuple[Path, ...] = (
+    Path("/var/run/docker.sock"),
+    Path("/run/docker.sock"),
+    Path("/var/snap/docker/current/run/docker.sock"),
+)
+
+WATCHDOG_LABEL = "watchdog"
+WATCHDOG_MANAGED = "true"

corekit/docker/disk/__init__.py

@@ -0,0 +1,8 @@
+"""
+Docker disk inspection: what is using the space under the Docker root.
+"""
+
+from corekit.docker.disk.inspector import DockerDiskInspector
+from corekit.docker.disk.models import ContainerLog, DfCategory, ImageSize
+
+__all__ = ["ContainerLog", "DfCategory", "DockerDiskInspector", "ImageSize"]

corekit/docker/disk/constants.py

@@ -0,0 +1,14 @@
+"""
+Constants for Docker disk inspection.
+"""
+
+# (display name, key in the daemon's df report, dot-path to the entry's size)
+# Volumes nest their size under UsageData; everything else is a top-level field.
+DF_CATEGORIES: tuple[tuple[str, str, str], ...] = (
+    ("Images", "Images", "Size"),
+    ("Containers", "Containers", "SizeRw"),
+    ("Volumes", "Volumes", "UsageData.Size"),
+    ("Build Cache", "BuildCache", "Size"),
+)
+
+UNREFERENCED_IMAGE = "<none>:<none>"

corekit/docker/disk/inspector.py

@@ -0,0 +1,217 @@
+"""
+Where Docker's disk went.
+
+``DockerDiskInspector`` is the read-side counterpart to ``Watchdog``:
+Watchdog changes container state, the inspector measures it -- Docker root
+usage, per-container log sizes, largest volumes and images, and a
+``docker system df``-style summary. It can also truncate container logs,
+which is a measurement remedy rather than lifecycle control, and safe:
+truncating in place reclaims space immediately, deleting would not.
+
+Daemon data comes from the Docker SDK (built by ``DockerClientProvider``,
+so snap and classic installs are both found); on-disk sizes come from
+``corekit.files.DiskScanner``, because the daemon does not report log-file
+sizes and its volume accounting does not match ``du``. Filesystem access
+under the Docker root typically needs elevation, which the scanner handles.
+
+Daemon payloads are untrusted-shaped: every field is extracted through the
+``parse_*``/``safe_*`` utilities and ``getval`` rather than raw indexing.
+
+    inspector = DockerDiskInspector()
+    inspector.root_dir()          # snap-aware: the socket is discovered
+    inspector.container_logs()    # every log, biggest first
+    inspector.truncate_log("api") # after you have looked at it
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from docker import DockerClient
+from docker.errors import APIError, NotFound
+
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.disk import constants as disk_constants
+from corekit.docker.disk.models import ContainerLog, DfCategory, ImageSize
+from corekit.files.disk import DiskScanner, PathSize
+from corekit.observability.benchmarkable import Benchmarkable
+from corekit.utils import first_string, getval, parse_dict, safe_dict, safe_int, safe_list, safe_string
+
+__all__ = ["DockerDiskInspector"]
+
+
+class DockerDiskInspector(Benchmarkable):
+    """
+    Measures Docker disk usage via the SDK plus filesystem scans.
+
+    Unlike Watchdog there is no label enforcement: measuring is read-only,
+    and the one mutating operation (log truncation) is addressed by exact
+    container name or ID, never by pattern.
+    """
+
+    def __init__(
+        self,
+        docker_host: str | None = None,
+        scanner: DiskScanner | None = None,
+    ) -> None:
+        """
+        :param docker_host: Docker endpoint; resolved by
+            ``DockerClientProvider`` (``$DOCKER_HOST``, existing sockets
+            including snap's, then the classic default) when omitted.
+        :param scanner: filesystem scanner for on-disk sizes; one with
+            elevation enabled by default.
+        """
+        super().__init__()
+        self.docker_host = DockerClientProvider.resolve_host(docker_host)
+        self._scanner = scanner or DiskScanner()
+        self._client_instance: DockerClient | None = None
+
+    @property
+    def _client(self) -> DockerClient:
+        """
+        The Docker client, connected on first use.
+        """
+        if self._client_instance is None:
+            self._client_instance = DockerClientProvider.create_client(self.docker_host)
+        return self._client_instance
+
+    # =============
+    # Daemon-level queries
+    # =============
+
+    def root_dir(self) -> Path:
+        """
+        The daemon's DockerRootDir (e.g. /var/snap/docker/common/var-lib-docker).
+        """
+        root = safe_string(getval(safe_dict(self._client.info()), "DockerRootDir"))
+        if not root:
+            raise RuntimeError("Daemon did not report a DockerRootDir (is docker running?)")
+        return Path(root)
+
+    def df_summary(self) -> list[DfCategory]:
+        """
+        Aggregate the daemon's df report into per-category counts and sizes.
+        """
+        report = parse_dict(self._client.df()).value
+        categories: list[DfCategory] = []
+        for display_name, report_key, size_path in disk_constants.DF_CATEGORIES:
+            entries = safe_list(getval(report, report_key, default=[]))
+            total = sum(safe_int(getval(entry, size_path, default=0)) for entry in entries)
+            categories.append(DfCategory(name=display_name, count=len(entries), size_bytes=total))
+        return categories
+
+    def images(self) -> list[ImageSize]:
+        """
+        Every image with its daemon-reported size, biggest first.
+        """
+        images = [
+            ImageSize(
+                reference=first_string(safe_list(image.tags), disk_constants.UNREFERENCED_IMAGE),
+                image_id=safe_string(image.short_id),
+                size_bytes=safe_int(getval(parse_dict(image.attrs).value, "Size")),
+            )
+            for image in self._client.images.list()
+        ]
+        return sorted(images, key=lambda img: img.size_bytes, reverse=True)
+
+    # =============
+    # Filesystem-backed queries
+    # =============
+
+    def root_children(self, root: Path | None = None) -> list[PathSize]:
+        """
+        Largest directories directly under the Docker root.
+        """
+        return self._scanner.child_sizes(root or self.root_dir())
+
+    def volumes(self, root: Path | None = None) -> list[PathSize]:
+        """
+        Largest Docker volumes (by on-disk size), biggest first.
+        """
+        return self._scanner.child_sizes((root or self.root_dir()) / "volumes")
+
+    def container_logs(self) -> list[ContainerLog]:
+        """
+        Every container's log with its size, biggest first.
+
+        Containers without a log path (non json-file drivers) or with an
+        unmeasurable log file are skipped.
+        """
+        logs: list[ContainerLog] = []
+        try:
+            containers = self._client.containers.list(all=True)
+        except APIError as exc:
+            self.error(f"Docker API error listing containers: {exc}")
+            return []
+        for container in containers:
+            log = self._measure_container_log(container)
+            if log is not None:
+                logs.append(log)
+        return sorted(logs, key=lambda x: x.size_bytes, reverse=True)
+
+    def logs_above(self, threshold_bytes: int) -> list[ContainerLog]:
+        """
+        Container logs strictly larger than the threshold, biggest first.
+        """
+        return [log for log in self.container_logs() if log.size_bytes > threshold_bytes]
+
+    def log_for(self, identifier: str) -> ContainerLog:
+        """
+        Resolve one container by name or ID and measure its log.
+
+        Raises LookupError if the container does not exist or has no log.
+        """
+        try:
+            container = self._client.containers.get(identifier)
+        except NotFound as exc:
+            raise LookupError(f"Container not found: {identifier}") from exc
+
+        log = self._measure_container_log(container)
+        if log is None:
+            raise LookupError(f"Container has no measurable log file (non json-file driver?): {identifier}")
+        return log
+
+    # =============
+    # Truncation
+    # =============
+
+    def truncate_log(self, identifier: str) -> ContainerLog:
+        """
+        Truncate one container's log; returns its pre-truncate measurement.
+        """
+        log = self.log_for(identifier)
+        self._scanner.truncate(log.log_path)
+        return log
+
+    def truncate_logs(self, logs: list[ContainerLog]) -> None:
+        """
+        Truncate each given container log.
+        """
+        for log in logs:
+            self._scanner.truncate(log.log_path)
+
+    # =============
+    # Private Methods
+    # =============
+
+    def _measure_container_log(self, container: object) -> ContainerLog | None:
+        """
+        Build a ContainerLog for one SDK container, or None when it has no
+        log path or the file cannot be measured.
+        """
+
+        log_path_text = safe_string(getval(container, "attrs.LogPath", default=""))
+        if not log_path_text:
+            return None
+
+        log_path = Path(log_path_text)
+        size = self._scanner.file_size(log_path)
+        if size is None:
+            return None
+
+        return ContainerLog(
+            container_id=safe_string(getval(container, "id")),
+            name=safe_string(getval(container, "name")).lstrip("/"),
+            log_path=log_path,
+            size_bytes=size,
+        )

corekit/docker/disk/models.py

@@ -0,0 +1,44 @@
+"""
+Data shapes returned by DockerDiskInspector.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+__all__ = ["ContainerLog", "DfCategory", "ImageSize"]
+
+
+@dataclass(frozen=True)
+class ContainerLog:
+    """
+    One container's json-file log and its measured size.
+    """
+
+    container_id: str
+    name: str
+    log_path: Path
+    size_bytes: int
+
+
+@dataclass(frozen=True)
+class ImageSize:
+    """
+    One image and its size as reported by the daemon.
+    """
+
+    reference: str
+    image_id: str
+    size_bytes: int
+
+
+@dataclass(frozen=True)
+class DfCategory:
+    """
+    One row of a ``docker system df``-style summary.
+    """
+
+    name: str
+    count: int
+    size_bytes: int

corekit/docker/watchdog.py

@@ -18,13 +18,13 @@ from docker import DockerClient
 from docker.errors import APIError, NotFound
 from docker.models.containers import Container
 
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.constants import DEFAULT_DOCKER_HOST, WATCHDOG_LABEL, WATCHDOG_MANAGED
 from corekit.observability.benchmarkable import Benchmarkable
 
-__all__ = ["Watchdog"]
-
-DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
-WATCHDOG_LABEL = "watchdog"
-WATCHDOG_MANAGED = "true"
+# DEFAULT_DOCKER_HOST / WATCHDOG_* are re-exported for backwards
+# compatibility; the canonical definitions live in corekit.docker.constants.
+__all__ = ["DEFAULT_DOCKER_HOST", "WATCHDOG_LABEL", "WATCHDOG_MANAGED", "Watchdog"]
 
 
 def _accept_all(*args: Any, **kwargs: Any) -> bool:
@@ -44,13 +44,15 @@ class Watchdog(Benchmarkable):
 
     def __init__(self, docker_host: str | None = None, enforce_label: bool = True) -> None:
         """
-        :param docker_host: Docker endpoint. Defaults to the local socket.
+        :param docker_host: Docker endpoint. Defaults to the endpoint
+            resolved by ``DockerClientProvider`` (``$DOCKER_HOST``, then an
+            existing socket -- snap included -- then the classic default).
         :param enforce_label: restrict actions to containers carrying the
             managed label. On by default; turn it off only when this watchdog
             is meant to control every container on the host.
         """
         super().__init__()
-        self.docker_host = docker_host or DEFAULT_DOCKER_HOST
+        self.docker_host = DockerClientProvider.resolve_host(docker_host)
         self.enforce_label = enforce_label
         self._client_instance: DockerClient | None = None
 
@@ -63,7 +65,7 @@ class Watchdog(Benchmarkable):
         and inspected on a machine where Docker is not running.
         """
         if self._client_instance is None:
-            self._client_instance = DockerClient(base_url=self.docker_host)
+            self._client_instance = DockerClientProvider.create_client(self.docker_host)
         return self._client_instance
 
     @staticmethod

corekit/exceptions/__init__.py

@@ -7,6 +7,7 @@ a user. ``CoreHTTPException`` is the FastAPI-facing public exception.
 """
 
 from corekit.exceptions.base import (
+    CommandFailedError,
     CoreException,
     CoreHTTPException,
     ExponentialBackoffTimeoutException,
@@ -20,6 +21,7 @@ from corekit.exceptions.types import ArbitraryCoreExceptionType, CoreExceptionTy
 
 __all__ = [
     "ArbitraryCoreExceptionType",
+    "CommandFailedError",
     "CoreException",
     "CoreExceptionType",
     "CoreHTTPException",

corekit/exceptions/base.py

@@ -27,6 +27,7 @@ from fastapi import HTTPException
 from corekit.exceptions.enum import Retryability
 
 __all__ = [
+    "CommandFailedError",
     "CoreException",
     "CoreHTTPException",
     "RetryableCoreHTTPException",
@@ -112,6 +113,24 @@ class ExponentialBackoffTimeoutException(InternalCoreException):
         self.attempts = attempts
 
 
+class CommandFailedError(InternalCoreException):
+    """
+    An external command exited non-zero, timed out, or could not be started.
+
+    ``command`` is the argv that was run; ``detail`` (also carried as the
+    shared ``error`` field) is stderr text or the OS-level reason.
+    """
+
+    def __init__(self, command: list[str], detail: str) -> None:
+        super().__init__(
+            message=f"Command failed: {' '.join(command)}",
+            retryable=Retryability.NON_RETRYABLE,
+            error=detail,
+        )
+        self.command = command
+        self.detail = detail
+
+
 class CoreHTTPException(HTTPException, PublicCoreException):
     """
     FastAPI HTTPException that is also a PublicCoreException.

corekit/files/__init__.py

@@ -9,15 +9,18 @@ File reading and writing, with format handled by the manager.
 """
 
 from corekit.files.base import FileContent, FileError, FileManager
+from corekit.files.disk import DiskScanner, PathSize
 from corekit.files.enum import FileMode
 from corekit.files.json import JsonFileManager
 from corekit.files.toml import TomlFileManager
 
 __all__ = [
+    "DiskScanner",
     "FileContent",
     "FileError",
     "FileManager",
     "FileMode",
     "JsonFileManager",
+    "PathSize",
     "TomlFileManager",
 ]

corekit/files/disk.py

@@ -0,0 +1,123 @@
+"""
+Measuring and truncating files on disk.
+
+``DiskScanner`` answers "how big is this path?" for files and directory
+trees, and truncates files in place. Every operation tries direct access
+first and, on failure, retries elevated through the ``CommandRunner``
+(log files and layer directories under a Docker root are root-owned, so an
+unprivileged process typically cannot stat them). Pass ``elevate=False``
+to disable the retry; unmeasurable paths then report ``None`` instead of
+prompting for a password.
+
+Sizes for single files are apparent sizes (``stat``); directory sizes are
+``du -sb`` totals. Both are byte counts and directly comparable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from corekit.exceptions import CommandFailedError
+from corekit.process import CommandRunner
+
+__all__ = ["DiskScanner", "PathSize"]
+
+
+@dataclass(frozen=True)
+class PathSize:
+    """
+    A filesystem path with its measured size in bytes.
+    """
+
+    path: Path
+    size_bytes: int
+
+
+class DiskScanner:
+    """
+    Filesystem sizing and in-place truncation, with optional elevation.
+    """
+
+    def __init__(self, runner: CommandRunner | None = None, *, elevate: bool = True) -> None:
+        """
+        :param runner: command executor for the elevated fallbacks and for
+            ``du``/``find``; a plain ``CommandRunner`` by default.
+        :param elevate: allow sudo retries when direct access fails.
+        """
+        self._runner = runner or CommandRunner()
+        self._elevate = elevate
+
+    def file_size(self, path: Path) -> int | None:
+        """
+        Size of a single file, or None if it cannot be measured.
+        """
+        try:
+            return path.stat().st_size
+        except OSError:
+            pass
+        if not self._elevate:
+            return None
+        try:
+            output = self._runner.run(["stat", "-c", "%s", str(path)], elevate=True)
+            return int(output.strip())
+        except (CommandFailedError, ValueError):
+            return None
+
+    def dir_size(self, path: Path) -> int | None:
+        """
+        Recursive size of a path (``du -sb``), or None if unmeasurable.
+        """
+        for elevate in (False, True):
+            if elevate and not self._elevate:
+                break
+            try:
+                output = self._runner.run(["du", "-sb", "--", str(path)], elevate=elevate)
+                return int(output.split()[0])
+            except (CommandFailedError, ValueError, IndexError):
+                continue
+        return None
+
+    def child_sizes(self, parent: Path) -> list[PathSize]:
+        """
+        Recursive sizes of every direct child of ``parent``, biggest first.
+        """
+        sizes: list[PathSize] = []
+        for child in self._children(parent):
+            size = self.dir_size(child)
+            if size is not None:
+                sizes.append(PathSize(path=child, size_bytes=size))
+        return sorted(sizes, key=lambda entry: entry.size_bytes, reverse=True)
+
+    def truncate(self, path: Path) -> None:
+        """
+        Truncate a file to zero bytes in place.
+
+        In place matters: deleting a file another process holds open does
+        not reclaim its space, truncating it does.
+        """
+        try:
+            with path.open("r+b") as handle:
+                handle.truncate(0)
+            return
+        except OSError:
+            pass
+        if not self._elevate:
+            raise PermissionError(f"Cannot truncate {path} (elevation disabled)")
+        self._runner.run(["truncate", "-s", "0", str(path)], elevate=True)
+
+    def _children(self, parent: Path) -> list[Path]:
+        """
+        Direct children of ``parent``, via elevated find if listing is denied.
+        """
+        try:
+            return sorted(parent.iterdir())
+        except OSError:
+            pass
+        if not self._elevate:
+            return []
+        try:
+            output = self._runner.run(["find", str(parent), "-mindepth", "1", "-maxdepth", "1"], elevate=True)
+        except CommandFailedError:
+            return []
+        return sorted(Path(line) for line in output.splitlines() if line.strip())

corekit/html/__init__.py

@@ -0,0 +1,22 @@
+"""
+HTML parsing and crawling helpers.
+"""
+
+from corekit.html.client import WebscrapeClient
+from corekit.html.models import LinkItem, Page, PageMetadata
+from corekit.html.parser import HtmlParser
+from corekit.html.scraper import Scraper
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.html.state import ScraperState
+
+__all__ = [
+    "CleanSettings",
+    "HtmlParser",
+    "LinkFilterSettings",
+    "LinkItem",
+    "Page",
+    "PageMetadata",
+    "Scraper",
+    "ScraperState",
+    "WebscrapeClient",
+]

corekit/html/client.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from corekit.http import BaseHttpClient
+
+
+class WebscrapeClient(BaseHttpClient):
+    """
+    HTTP client with browser-like default headers for scraping.
+
+    Inherits GET-only ``supported_methods`` from ``BaseHttpClient``.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        kwargs.setdefault("strict", False)
+        super().__init__(**kwargs)
+
+    @property
+    def headers(self) -> dict[str, str]:
+        return {
+            "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0",
+        }