python-corekit 0.4.1__tar.gz → 0.6.0__tar.gz

{python_corekit-0.4.1/python_corekit.egg-info → python_corekit-0.6.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: python-corekit
-Version: 0.4.1
-Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, and LLM chat/tool primitives
+Version: 0.6.0
+Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, LLM chat/tool primitives, and HTML scraping
 Author: Steven Jacobsen
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/stevejaker/corekit
@@ -23,6 +23,7 @@ Requires-Dist: sqlmodel<0.1,>=0.0.16
 Requires-Dist: SQLAlchemy<3,>=2.0
 Requires-Dist: redis<7,>=5.0
 Requires-Dist: httpx<1,>=0.27
+Requires-Dist: selectolax<1,>=0.3
 Requires-Dist: docker<8,>=7.0
 Requires-Dist: PyYAML<7,>=6.0
 Requires-Dist: dill<0.5,>=0.3.8
@@ -347,13 +348,13 @@ class GithubClient(BaseApiClient):
         return "https://api.github.com"
 
 response = GithubClient().get("/users/octocat")
-response.data["login"]
+response.data.login
 ```
 
-Retries 429 and 5xx with exponential backoff. Every response is a
-`BaseApiResponse`, so a non-JSON error page leaves `data` empty rather than
-raising. The same methods are awaitable inside a running event loop:
-`await client.get(...)`.
+Retries 429 and 5xx with exponential backoff. ``BaseHttpClient`` returns a
+``BaseHttpResponse`` (text + metadata); ``BaseApiClient`` adds structured
+``data`` from JSON (``None`` when the body is not JSON). The same methods are
+awaitable inside a running event loop: `await client.get(...)`.
 
 ## Serialization
 
@@ -432,6 +433,7 @@ corekit/
 
   observability/                   Loggable, Benchmarkable, Timer
   registry/  schemas/  utils/      registries, enums and fields, helpers
+  process/                         CommandRunner (external commands)
   data/                            Dataset and its filter expressions
   jobs/                            queue-independent background tasks
   crypto/  files/  serialization/
@@ -444,7 +446,7 @@ corekit/
   http/                            BaseHttpClient, BaseApiClient, retries, responses
 
   api/                             Application, lifespan, middleware, routers
-  docker/  notifications/  etl/
+  docker/  notifications/  etl/    Watchdog, DockerDiskInspector, client provider
   llm/                             messages, tools, agentic loop (no GenerationClient)
 
   events/  log_monitor/            built on the capabilities above

{python_corekit-0.4.1 → python_corekit-0.6.0}/README.md

@@ -311,13 +311,13 @@ class GithubClient(BaseApiClient):
         return "https://api.github.com"
 
 response = GithubClient().get("/users/octocat")
-response.data["login"]
+response.data.login
 ```
 
-Retries 429 and 5xx with exponential backoff. Every response is a
-`BaseApiResponse`, so a non-JSON error page leaves `data` empty rather than
-raising. The same methods are awaitable inside a running event loop:
-`await client.get(...)`.
+Retries 429 and 5xx with exponential backoff. ``BaseHttpClient`` returns a
+``BaseHttpResponse`` (text + metadata); ``BaseApiClient`` adds structured
+``data`` from JSON (``None`` when the body is not JSON). The same methods are
+awaitable inside a running event loop: `await client.get(...)`.
 
 ## Serialization
 
@@ -396,6 +396,7 @@ corekit/
 
   observability/                   Loggable, Benchmarkable, Timer
   registry/  schemas/  utils/      registries, enums and fields, helpers
+  process/                         CommandRunner (external commands)
   data/                            Dataset and its filter expressions
   jobs/                            queue-independent background tasks
   crypto/  files/  serialization/
@@ -408,7 +409,7 @@ corekit/
   http/                            BaseHttpClient, BaseApiClient, retries, responses
 
   api/                             Application, lifespan, middleware, routers
-  docker/  notifications/  etl/
+  docker/  notifications/  etl/    Watchdog, DockerDiskInspector, client provider
   llm/                             messages, tools, agentic loop (no GenerationClient)
 
   events/  log_monitor/            built on the capabilities above

{python_corekit-0.4.1 → python_corekit-0.6.0}/corekit/connections/sql/connection.py

@@ -330,15 +330,19 @@ class SQLConnection(Connectable):
         """
         return list(self._exec(query.build()))
 
-    def delete_many_by_query(self, query: Query) -> None:
+    def delete_many_by_query(self, query: Query) -> int:
         """
         Delete every row matching a Query.
+
+        Returns the number of rows deleted.
         """
         self.require_writable()
         self._start_session()
-        for item in self.fetch_many_by_query(query):
+        items = self.fetch_many_by_query(query)
+        for item in items:
             self.session.delete(item)
         self.session.commit()
+        return len(items)
 
     def execute(self, operation: Operation) -> Any:
         """

python_corekit-0.6.0/corekit/docker/__init__.py

@@ -0,0 +1,9 @@
+"""
+Docker container control and disk inspection.
+"""
+
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.disk import ContainerLog, DfCategory, DockerDiskInspector, ImageSize
+from corekit.docker.watchdog import Watchdog
+
+__all__ = ["ContainerLog", "DfCategory", "DockerClientProvider", "DockerDiskInspector", "ImageSize", "Watchdog"]

python_corekit-0.6.0/corekit/docker/client.py

@@ -0,0 +1,48 @@
+"""
+Getting a Docker SDK client without assuming where the daemon lives.
+
+``DockerClientProvider`` is the one place that resolves the endpoint:
+an explicit host wins, then ``$DOCKER_HOST``, then the first socket that
+actually exists among the usual locations (including snap's), and only
+then the classic default. ``Watchdog`` and ``DockerDiskInspector`` both
+build their clients through here, so a daemon that moves (classic install
+to snap, say) is found by both.
+"""
+
+from __future__ import annotations
+
+import os
+
+from docker import DockerClient
+
+from corekit.docker.constants import DEFAULT_DOCKER_HOST, DOCKER_SOCKET_CANDIDATES
+
+__all__ = ["DockerClientProvider"]
+
+
+class DockerClientProvider:
+    """
+    Resolves Docker endpoints and builds SDK clients for them.
+    """
+
+    @staticmethod
+    def resolve_host(docker_host: str | None = None) -> str:
+        """
+        Resolve the endpoint to use, without connecting to it.
+        """
+        if docker_host:
+            return docker_host
+        from_env = os.environ.get("DOCKER_HOST")
+        if from_env:
+            return from_env
+        for candidate in DOCKER_SOCKET_CANDIDATES:
+            if candidate.exists():
+                return f"unix://{candidate}"
+        return DEFAULT_DOCKER_HOST
+
+    @staticmethod
+    def create_client(docker_host: str | None = None) -> DockerClient:
+        """
+        Build an SDK client for the resolved endpoint.
+        """
+        return DockerClient(base_url=DockerClientProvider.resolve_host(docker_host))

python_corekit-0.6.0/corekit/docker/constants.py

@@ -0,0 +1,19 @@
+"""
+Shared Docker daemon constants.
+"""
+
+from pathlib import Path
+
+DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
+
+# Candidate sockets in preference order. Snap Docker does not use
+# /var/run/docker.sock, which is the single most common reason an SDK
+# client "cannot connect" on an Ubuntu host.
+DOCKER_SOCKET_CANDIDATES: tuple[Path, ...] = (
+    Path("/var/run/docker.sock"),
+    Path("/run/docker.sock"),
+    Path("/var/snap/docker/current/run/docker.sock"),
+)
+
+WATCHDOG_LABEL = "watchdog"
+WATCHDOG_MANAGED = "true"

python_corekit-0.6.0/corekit/docker/disk/__init__.py

@@ -0,0 +1,8 @@
+"""
+Docker disk inspection: what is using the space under the Docker root.
+"""
+
+from corekit.docker.disk.inspector import DockerDiskInspector
+from corekit.docker.disk.models import ContainerLog, DfCategory, ImageSize
+
+__all__ = ["ContainerLog", "DfCategory", "DockerDiskInspector", "ImageSize"]

python_corekit-0.6.0/corekit/docker/disk/constants.py

@@ -0,0 +1,14 @@
+"""
+Constants for Docker disk inspection.
+"""
+
+# (display name, key in the daemon's df report, dot-path to the entry's size)
+# Volumes nest their size under UsageData; everything else is a top-level field.
+DF_CATEGORIES: tuple[tuple[str, str, str], ...] = (
+    ("Images", "Images", "Size"),
+    ("Containers", "Containers", "SizeRw"),
+    ("Volumes", "Volumes", "UsageData.Size"),
+    ("Build Cache", "BuildCache", "Size"),
+)
+
+UNREFERENCED_IMAGE = "<none>:<none>"

python_corekit-0.6.0/corekit/docker/disk/inspector.py

@@ -0,0 +1,217 @@
+"""
+Where Docker's disk went.
+
+``DockerDiskInspector`` is the read-side counterpart to ``Watchdog``:
+Watchdog changes container state, the inspector measures it -- Docker root
+usage, per-container log sizes, largest volumes and images, and a
+``docker system df``-style summary. It can also truncate container logs,
+which is a measurement remedy rather than lifecycle control, and safe:
+truncating in place reclaims space immediately, deleting would not.
+
+Daemon data comes from the Docker SDK (built by ``DockerClientProvider``,
+so snap and classic installs are both found); on-disk sizes come from
+``corekit.files.DiskScanner``, because the daemon does not report log-file
+sizes and its volume accounting does not match ``du``. Filesystem access
+under the Docker root typically needs elevation, which the scanner handles.
+
+Daemon payloads are untrusted-shaped: every field is extracted through the
+``parse_*``/``safe_*`` utilities and ``getval`` rather than raw indexing.
+
+    inspector = DockerDiskInspector()
+    inspector.root_dir()          # snap-aware: the socket is discovered
+    inspector.container_logs()    # every log, biggest first
+    inspector.truncate_log("api") # after you have looked at it
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from docker import DockerClient
+from docker.errors import APIError, NotFound
+
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.disk import constants as disk_constants
+from corekit.docker.disk.models import ContainerLog, DfCategory, ImageSize
+from corekit.files.disk import DiskScanner, PathSize
+from corekit.observability.benchmarkable import Benchmarkable
+from corekit.utils import first_string, getval, parse_dict, safe_dict, safe_int, safe_list, safe_string
+
+__all__ = ["DockerDiskInspector"]
+
+
+class DockerDiskInspector(Benchmarkable):
+    """
+    Measures Docker disk usage via the SDK plus filesystem scans.
+
+    Unlike Watchdog there is no label enforcement: measuring is read-only,
+    and the one mutating operation (log truncation) is addressed by exact
+    container name or ID, never by pattern.
+    """
+
+    def __init__(
+        self,
+        docker_host: str | None = None,
+        scanner: DiskScanner | None = None,
+    ) -> None:
+        """
+        :param docker_host: Docker endpoint; resolved by
+            ``DockerClientProvider`` (``$DOCKER_HOST``, existing sockets
+            including snap's, then the classic default) when omitted.
+        :param scanner: filesystem scanner for on-disk sizes; one with
+            elevation enabled by default.
+        """
+        super().__init__()
+        self.docker_host = DockerClientProvider.resolve_host(docker_host)
+        self._scanner = scanner or DiskScanner()
+        self._client_instance: DockerClient | None = None
+
+    @property
+    def _client(self) -> DockerClient:
+        """
+        The Docker client, connected on first use.
+        """
+        if self._client_instance is None:
+            self._client_instance = DockerClientProvider.create_client(self.docker_host)
+        return self._client_instance
+
+    # =============
+    # Daemon-level queries
+    # =============
+
+    def root_dir(self) -> Path:
+        """
+        The daemon's DockerRootDir (e.g. /var/snap/docker/common/var-lib-docker).
+        """
+        root = safe_string(getval(safe_dict(self._client.info()), "DockerRootDir"))
+        if not root:
+            raise RuntimeError("Daemon did not report a DockerRootDir (is docker running?)")
+        return Path(root)
+
+    def df_summary(self) -> list[DfCategory]:
+        """
+        Aggregate the daemon's df report into per-category counts and sizes.
+        """
+        report = parse_dict(self._client.df()).value
+        categories: list[DfCategory] = []
+        for display_name, report_key, size_path in disk_constants.DF_CATEGORIES:
+            entries = safe_list(getval(report, report_key, default=[]))
+            total = sum(safe_int(getval(entry, size_path, default=0)) for entry in entries)
+            categories.append(DfCategory(name=display_name, count=len(entries), size_bytes=total))
+        return categories
+
+    def images(self) -> list[ImageSize]:
+        """
+        Every image with its daemon-reported size, biggest first.
+        """
+        images = [
+            ImageSize(
+                reference=first_string(safe_list(image.tags), disk_constants.UNREFERENCED_IMAGE),
+                image_id=safe_string(image.short_id),
+                size_bytes=safe_int(getval(parse_dict(image.attrs).value, "Size")),
+            )
+            for image in self._client.images.list()
+        ]
+        return sorted(images, key=lambda img: img.size_bytes, reverse=True)
+
+    # =============
+    # Filesystem-backed queries
+    # =============
+
+    def root_children(self, root: Path | None = None) -> list[PathSize]:
+        """
+        Largest directories directly under the Docker root.
+        """
+        return self._scanner.child_sizes(root or self.root_dir())
+
+    def volumes(self, root: Path | None = None) -> list[PathSize]:
+        """
+        Largest Docker volumes (by on-disk size), biggest first.
+        """
+        return self._scanner.child_sizes((root or self.root_dir()) / "volumes")
+
+    def container_logs(self) -> list[ContainerLog]:
+        """
+        Every container's log with its size, biggest first.
+
+        Containers without a log path (non json-file drivers) or with an
+        unmeasurable log file are skipped.
+        """
+        logs: list[ContainerLog] = []
+        try:
+            containers = self._client.containers.list(all=True)
+        except APIError as exc:
+            self.error(f"Docker API error listing containers: {exc}")
+            return []
+        for container in containers:
+            log = self._measure_container_log(container)
+            if log is not None:
+                logs.append(log)
+        return sorted(logs, key=lambda x: x.size_bytes, reverse=True)
+
+    def logs_above(self, threshold_bytes: int) -> list[ContainerLog]:
+        """
+        Container logs strictly larger than the threshold, biggest first.
+        """
+        return [log for log in self.container_logs() if log.size_bytes > threshold_bytes]
+
+    def log_for(self, identifier: str) -> ContainerLog:
+        """
+        Resolve one container by name or ID and measure its log.
+
+        Raises LookupError if the container does not exist or has no log.
+        """
+        try:
+            container = self._client.containers.get(identifier)
+        except NotFound as exc:
+            raise LookupError(f"Container not found: {identifier}") from exc
+
+        log = self._measure_container_log(container)
+        if log is None:
+            raise LookupError(f"Container has no measurable log file (non json-file driver?): {identifier}")
+        return log
+
+    # =============
+    # Truncation
+    # =============
+
+    def truncate_log(self, identifier: str) -> ContainerLog:
+        """
+        Truncate one container's log; returns its pre-truncate measurement.
+        """
+        log = self.log_for(identifier)
+        self._scanner.truncate(log.log_path)
+        return log
+
+    def truncate_logs(self, logs: list[ContainerLog]) -> None:
+        """
+        Truncate each given container log.
+        """
+        for log in logs:
+            self._scanner.truncate(log.log_path)
+
+    # =============
+    # Private Methods
+    # =============
+
+    def _measure_container_log(self, container: object) -> ContainerLog | None:
+        """
+        Build a ContainerLog for one SDK container, or None when it has no
+        log path or the file cannot be measured.
+        """
+
+        log_path_text = safe_string(getval(container, "attrs.LogPath", default=""))
+        if not log_path_text:
+            return None
+
+        log_path = Path(log_path_text)
+        size = self._scanner.file_size(log_path)
+        if size is None:
+            return None
+
+        return ContainerLog(
+            container_id=safe_string(getval(container, "id")),
+            name=safe_string(getval(container, "name")).lstrip("/"),
+            log_path=log_path,
+            size_bytes=size,
+        )

python_corekit-0.6.0/corekit/docker/disk/models.py

@@ -0,0 +1,44 @@
+"""
+Data shapes returned by DockerDiskInspector.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+__all__ = ["ContainerLog", "DfCategory", "ImageSize"]
+
+
+@dataclass(frozen=True)
+class ContainerLog:
+    """
+    One container's json-file log and its measured size.
+    """
+
+    container_id: str
+    name: str
+    log_path: Path
+    size_bytes: int
+
+
+@dataclass(frozen=True)
+class ImageSize:
+    """
+    One image and its size as reported by the daemon.
+    """
+
+    reference: str
+    image_id: str
+    size_bytes: int
+
+
+@dataclass(frozen=True)
+class DfCategory:
+    """
+    One row of a ``docker system df``-style summary.
+    """
+
+    name: str
+    count: int
+    size_bytes: int

{python_corekit-0.4.1 → python_corekit-0.6.0}/corekit/docker/watchdog.py

@@ -18,13 +18,13 @@ from docker import DockerClient
 from docker.errors import APIError, NotFound
 from docker.models.containers import Container
 
+from corekit.docker.client import DockerClientProvider
+from corekit.docker.constants import DEFAULT_DOCKER_HOST, WATCHDOG_LABEL, WATCHDOG_MANAGED
 from corekit.observability.benchmarkable import Benchmarkable
 
-__all__ = ["Watchdog"]
-
-DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
-WATCHDOG_LABEL = "watchdog"
-WATCHDOG_MANAGED = "true"
+# DEFAULT_DOCKER_HOST / WATCHDOG_* are re-exported for backwards
+# compatibility; the canonical definitions live in corekit.docker.constants.
+__all__ = ["DEFAULT_DOCKER_HOST", "WATCHDOG_LABEL", "WATCHDOG_MANAGED", "Watchdog"]
 
 
 def _accept_all(*args: Any, **kwargs: Any) -> bool:
@@ -44,13 +44,15 @@ class Watchdog(Benchmarkable):
 
     def __init__(self, docker_host: str | None = None, enforce_label: bool = True) -> None:
         """
-        :param docker_host: Docker endpoint. Defaults to the local socket.
+        :param docker_host: Docker endpoint. Defaults to the endpoint
+            resolved by ``DockerClientProvider`` (``$DOCKER_HOST``, then an
+            existing socket -- snap included -- then the classic default).
         :param enforce_label: restrict actions to containers carrying the
             managed label. On by default; turn it off only when this watchdog
             is meant to control every container on the host.
         """
         super().__init__()
-        self.docker_host = docker_host or DEFAULT_DOCKER_HOST
+        self.docker_host = DockerClientProvider.resolve_host(docker_host)
         self.enforce_label = enforce_label
         self._client_instance: DockerClient | None = None
 
@@ -63,7 +65,7 @@ class Watchdog(Benchmarkable):
         and inspected on a machine where Docker is not running.
         """
         if self._client_instance is None:
-            self._client_instance = DockerClient(base_url=self.docker_host)
+            self._client_instance = DockerClientProvider.create_client(self.docker_host)
         return self._client_instance
 
     @staticmethod

{python_corekit-0.4.1 → python_corekit-0.6.0}/corekit/exceptions/__init__.py

@@ -7,6 +7,7 @@ a user. ``CoreHTTPException`` is the FastAPI-facing public exception.
 """
 
 from corekit.exceptions.base import (
+    CommandFailedError,
     CoreException,
     CoreHTTPException,
     ExponentialBackoffTimeoutException,
@@ -20,6 +21,7 @@ from corekit.exceptions.types import ArbitraryCoreExceptionType, CoreExceptionTy
 
 __all__ = [
     "ArbitraryCoreExceptionType",
+    "CommandFailedError",
     "CoreException",
     "CoreExceptionType",
     "CoreHTTPException",

{python_corekit-0.4.1 → python_corekit-0.6.0}/corekit/exceptions/base.py

@@ -27,6 +27,7 @@ from fastapi import HTTPException
 from corekit.exceptions.enum import Retryability
 
 __all__ = [
+    "CommandFailedError",
     "CoreException",
     "CoreHTTPException",
     "RetryableCoreHTTPException",
@@ -112,6 +113,24 @@ class ExponentialBackoffTimeoutException(InternalCoreException):
         self.attempts = attempts
 
 
+class CommandFailedError(InternalCoreException):
+    """
+    An external command exited non-zero, timed out, or could not be started.
+
+    ``command`` is the argv that was run; ``detail`` (also carried as the
+    shared ``error`` field) is stderr text or the OS-level reason.
+    """
+
+    def __init__(self, command: list[str], detail: str) -> None:
+        super().__init__(
+            message=f"Command failed: {' '.join(command)}",
+            retryable=Retryability.NON_RETRYABLE,
+            error=detail,
+        )
+        self.command = command
+        self.detail = detail
+
+
 class CoreHTTPException(HTTPException, PublicCoreException):
     """
     FastAPI HTTPException that is also a PublicCoreException.

{python_corekit-0.4.1 → python_corekit-0.6.0}/corekit/files/__init__.py

@@ -9,15 +9,18 @@ File reading and writing, with format handled by the manager.
 """
 
 from corekit.files.base import FileContent, FileError, FileManager
+from corekit.files.disk import DiskScanner, PathSize
 from corekit.files.enum import FileMode
 from corekit.files.json import JsonFileManager
 from corekit.files.toml import TomlFileManager
 
 __all__ = [
+    "DiskScanner",
     "FileContent",
     "FileError",
     "FileManager",
     "FileMode",
     "JsonFileManager",
+    "PathSize",
     "TomlFileManager",
 ]