pyhusk 0.1.0__py3-none-any.whl

pyhusk/__init__.py

@@ -0,0 +1,3 @@
+"""Minimal Docker images for FastAPI monorepos."""
+
+__version__ = "0.1.0"

pyhusk/_probe.py

@@ -0,0 +1,186 @@
+"""Route and dependency probe. Runs inside the TARGET repository's interpreter.
+
+Import nothing from pyhusk here, and nothing outside the standard library. This
+file is handed to a foreign interpreter by absolute path, because pyhusk is not
+installed in the target's virtualenv.
+
+Output on stdout is a single JSON object. Anything diagnostic goes to stderr, so
+stdout stays parseable.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import inspect
+import json
+import sys
+from pathlib import Path
+
+
+def _source_file(obj: object) -> str | None:
+    """Best-effort source file for a callable, seeing through decorators."""
+    try:
+        unwrapped = inspect.unwrap(obj)
+    except Exception:
+        unwrapped = obj
+    try:
+        found = inspect.getsourcefile(unwrapped)
+    except (TypeError, OSError):
+        found = None
+    if found is None:
+        # A callable instance: fall back to the class that defines __call__.
+        try:
+            found = inspect.getsourcefile(unwrapped.__class__)
+        except (TypeError, OSError, AttributeError):
+            return None
+    return found
+
+
+def _collect_dependant(dependant: object, files: set[str], seen: set[int]) -> None:
+    """Walk a FastAPI dependant tree, recording the source of every callable.
+
+    FastAPI has already resolved Depends() into this tree at route-registration
+    time, including router-level and app-level dependencies, so walking it is
+    exact rather than heuristic.
+    """
+    if dependant is None or id(dependant) in seen:
+        return
+    seen.add(id(dependant))
+
+    call = getattr(dependant, "call", None)
+    if call is not None:
+        found = _source_file(call)
+        if found:
+            files.add(found)
+
+    for sub in getattr(dependant, "dependencies", None) or []:
+        _collect_dependant(sub, files, seen)
+
+
+def _collect_routes(
+    routes: object,
+    prefix: str,
+    files: set[str],
+    described: list[dict[str, object]],
+    seen: set[int],
+) -> None:
+    """Walk app.routes, which is NOT homogeneous.
+
+    Three shapes must survive here:
+      * fastapi.routing.APIRoute / APIWebSocketRoute - .endpoint and .dependant
+      * starlette.routing.Route / WebSocketRoute - .endpoint, NO .dependant,
+        because FastAPI runs no dependency injection for them
+      * starlette.routing.Mount - no .endpoint, but .routes to recurse into
+
+    Every attribute is read with getattr(..., None). Assuming APIRoute is how a
+    probe crashes on the first real repository it meets.
+    """
+    for route in routes or []:
+        if id(route) in seen:
+            continue
+        seen.add(id(route))
+
+        path = prefix + (getattr(route, "path", "") or "")
+
+        endpoint = getattr(route, "endpoint", None)
+        if endpoint is not None:
+            found = _source_file(endpoint)
+            if found:
+                files.add(found)
+
+        dependant = getattr(route, "dependant", None)
+        _collect_dependant(dependant, files, set())
+
+        nested = getattr(route, "routes", None)
+        if nested:
+            _collect_routes(nested, path, files, described, seen)
+            continue
+
+        if endpoint is None:
+            continue
+
+        methods = sorted(getattr(route, "methods", None) or [])
+        # Only APIRoute carries .dependant, and only a documented one appears in
+        # the OpenAPI schema. A plain Starlette route added with add_route never
+        # does, so verification must not demand it there.
+        documented = bool(
+            dependant is not None
+            and getattr(route, "include_in_schema", False)
+            and methods
+        )
+        described.append({"path": path, "methods": methods, "documented": documented})
+
+
+def _import_entry(entry: Path) -> object:
+    """Import the entry file as a module, so its relative imports resolve."""
+    repo_root = Path.cwd()
+    if str(repo_root) not in sys.path:
+        sys.path.insert(0, str(repo_root))
+    return importlib.import_module(".".join(entry.with_suffix("").parts))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="pyhusk route probe")
+    parser.add_argument("--entry", required=True, help="entry file, relative to repo root")
+    parser.add_argument("--app", required=True, help="attribute holding the app")
+    parser.add_argument(
+        "--factory",
+        action="store_true",
+        help="the attribute is a callable returning the app, not the app itself",
+    )
+    args = parser.parse_args()
+
+    module = _import_entry(Path(args.entry))
+
+    if not hasattr(module, args.app):
+        print(
+            f"module {module.__name__!r} has no attribute {args.app!r}; "
+            f"check the 'app' key for this service",
+            file=sys.stderr,
+        )
+        return 1
+
+    application = getattr(module, args.app)
+    if args.factory:
+        if not callable(application):
+            print(
+                f"{module.__name__}.{args.app} is not callable, but factory: true "
+                f"was configured for this service",
+                file=sys.stderr,
+            )
+            return 1
+        application = application()
+
+    if not hasattr(application, "routes"):
+        hint = "" if args.factory else ". If it is a factory function, set factory: true"
+        print(
+            f"{module.__name__}.{args.app} has no .routes attribute; it does not look "
+            f"like a FastAPI or Starlette application{hint}",
+            file=sys.stderr,
+        )
+        return 1
+
+    files: set[str] = set()
+    described: list[dict[str, object]] = []
+    _collect_routes(application.routes, "", files, described, set())
+
+    # Middleware, lifespan handlers and exception handlers are registered in the
+    # entry module and reachable from no route, so it is always part of the slice.
+    entry_source = _source_file(module)
+    if entry_source:
+        files.add(entry_source)
+
+    json.dump(
+        {
+            "files": sorted(str(Path(f).resolve()) for f in files),
+            "routes": described,
+            "openapi_url": getattr(application, "openapi_url", None),
+        },
+        sys.stdout,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyhusk/cli.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+from typing import Annotated
+
+import typer
+import yaml
+
+from pyhusk.config import (
+    CONFIG_NAME,
+    PyhuskConfig,
+    ServiceConfig,
+    autodetect_services,
+    load_config,
+)
+from pyhusk.deps import RequirementsResult, all_requirements
+from pyhusk.docker import image_exists, image_size, require_docker, run_docker
+from pyhusk.dockerize import assemble_context, render_base_dockerfile, render_dockerfile
+from pyhusk.errors import PyhuskError
+from pyhusk.pipeline import Resolution, is_stale, resolve
+from pyhusk.report import count_python_files, human_size
+from pyhusk.staleness import SLICE_LABEL
+from pyhusk.verify import verify
+
+app = typer.Typer(
+    add_completion=False,
+    help="Minimal Docker images for FastAPI monorepos.",
+    no_args_is_help=True,
+)
+
+RepoOption = Annotated[
+    Path,
+    typer.Option("--repo", help="Repository root. Defaults to the current directory."),
+]
+
+
+def _fail(message: str) -> None:
+    typer.secho(f"error: {message}", fg=typer.colors.RED, err=True)
+    raise typer.Exit(code=1)
+
+
+def _ensure_gitignored(repo: Path) -> None:
+    """Add .build/ to .gitignore exactly once."""
+    gitignore = repo / ".gitignore"
+    if not gitignore.exists():
+        gitignore.write_text(".build/\n", encoding="utf-8")
+        typer.echo("created .gitignore with .build/")
+        return
+    existing = gitignore.read_text(encoding="utf-8")
+    if any(line.strip() in {".build", ".build/"} for line in existing.splitlines()):
+        return
+    separator = "" if existing.endswith("\n") or not existing else "\n"
+    gitignore.write_text(f"{existing}{separator}.build/\n", encoding="utf-8")
+    typer.echo("added .build/ to .gitignore")
+
+
+@app.command()
+def init(repo: RepoOption = Path(".")) -> None:
+    """Write pyhusk.yaml from detected services and ignore the build directory."""
+    repo = repo.resolve()
+    config_path = repo / CONFIG_NAME
+
+    if config_path.exists():
+        typer.echo(f"{CONFIG_NAME} already exists, leaving it untouched.")
+    else:
+        try:
+            detected = autodetect_services(repo)
+        except PyhuskError as exc:
+            _fail(str(exc))
+        if not detected:
+            _fail(
+                "No main.py containing a FastAPI() call was found. Create your services "
+                f"first, or write {CONFIG_NAME} by hand."
+            )
+        document = {
+            "services": {
+                name: {
+                    "entry": str(service.entry.relative_to(repo)),
+                    "app": service.app,
+                }
+                for name, service in sorted(detected.items())
+            }
+        }
+        config_path.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8")
+        typer.echo(
+            f"wrote {CONFIG_NAME} with {len(detected)} service(s): "
+            f"{', '.join(sorted(detected))}"
+        )
+
+    _ensure_gitignored(repo)
+
+
+@app.command("list")
+def list_services(repo: RepoOption = Path(".")) -> None:
+    """List the services pyhusk can see."""
+    repo = repo.resolve()
+    try:
+        config = load_config(repo)
+    except PyhuskError as exc:
+        _fail(str(exc))
+    for name, service in sorted(config.services.items()):
+        relative = service.entry.relative_to(repo)
+        suffix = "()" if service.factory else ""
+        typer.echo(f"{name}\t{relative}\t{service.app}{suffix}")
+
+
+IncludeOption = Annotated[
+    list[Path] | None,
+    typer.Option("--include", help="Force a path into the slice. Repeatable."),
+]
+
+
+def _selected(config: PyhuskConfig, names: list[str] | None) -> dict[str, ServiceConfig]:
+    if not names:
+        return dict(config.services)
+    unknown = [name for name in names if name not in config.services]
+    if unknown:
+        _fail(
+            f"unknown service(s): {', '.join(unknown)}. "
+            f"Known: {', '.join(sorted(config.services))}"
+        )
+    return {name: config.services[name] for name in names}
+
+
+def _resolve_all(
+    repo: Path,
+    services: dict[str, ServiceConfig],
+    includes: list[Path],
+    shared_base: bool,
+) -> dict[str, Resolution]:
+    resolutions: dict[str, Resolution] = {}
+    for name, service in sorted(services.items()):
+        try:
+            resolutions[name] = resolve(repo, name, service, includes, shared_base)
+        except PyhuskError as exc:
+            _fail(f"{name}: {exc}")
+    return resolutions
+
+
+def _print_warnings(resolutions: dict[str, Resolution]) -> None:
+    for name, resolution in sorted(resolutions.items()):
+        for warning in resolution.warnings:
+            typer.secho(f"warning: {name}: {warning}", fg=typer.colors.YELLOW, err=True)
+
+
+def _build_bases(repo: Path, resolutions: dict[str, Resolution]) -> None:
+    """Build every shared base image the given resolutions need, once each.
+
+    Separate from service builds because `pyhusk plan --json` exists to fan out a
+    CI matrix, and two runners building the same base tag concurrently race. A
+    file lock cannot help across machines that share no filesystem, so CI runs
+    this as one job that the matrix depends on.
+    """
+    wanted: dict[str, RequirementsResult] = {}
+    sources: dict[str, str] = {}
+    for resolution in resolutions.values():
+        if resolution.base_tag is None:
+            continue
+        wanted[resolution.base_tag] = resolution.requirements
+        sources[resolution.base_tag] = resolution.base.pinned
+
+    for tag, requirements in sorted(wanted.items()):
+        if image_exists(tag):
+            typer.echo(f"base {tag} already present")
+            continue
+        context = repo / ".build" / "_bases" / tag.split(":", 1)[1]
+        if context.exists():
+            shutil.rmtree(context)
+        context.mkdir(parents=True)
+        (context / "requirements.txt").write_text(requirements.text, encoding="utf-8")
+        (context / "Dockerfile").write_text(
+            render_base_dockerfile(sources[tag]), encoding="utf-8"
+        )
+        typer.echo(f"building base {tag} ({len(requirements.pins)} packages)")
+        if run_docker(["build", "-t", tag, str(context)], capture_output=False).returncode != 0:
+            _fail(f"building base {tag} failed")
+
+
+@app.command("build-bases")
+def build_bases(
+    service: Annotated[
+        list[str] | None, typer.Argument(help="Limit to these services.")
+    ] = None,
+    repo: RepoOption = Path("."),
+) -> None:
+    """Build the shared base images the current plan requires."""
+    repo = repo.resolve()
+    try:
+        require_docker()
+        config = load_config(repo)
+    except PyhuskError as exc:
+        _fail(str(exc))
+    _build_bases(repo, _resolve_all(repo, _selected(config, service), [], True))
+
+
+def _compare_against_naive(repo: Path, resolution: Resolution) -> None:
+    """Build a whole-repo image the naive way and report the difference.
+
+    Doubles build time, so it is off by default. It exists so the size claim can
+    be checked rather than asserted.
+    """
+    context = repo / ".build" / "_naive" / resolution.name
+    if context.exists():
+        shutil.rmtree(context)
+    context.mkdir(parents=True)
+
+    for source in repo.rglob("*"):
+        if not source.is_file():
+            continue
+        relative = source.relative_to(repo)
+        if relative.parts and relative.parts[0] in {".build", ".venv", ".git"}:
+            continue
+        destination = context / relative
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, destination)
+
+    (context / "requirements.txt").write_text(all_requirements(repo).text, encoding="utf-8")
+    (context / "Dockerfile").write_text(
+        render_dockerfile(repo, resolution.service, resolution.base.pinned, True),
+        encoding="utf-8",
+    )
+
+    tag = f"{resolution.name}-naive:latest"
+    typer.echo(f"{resolution.name}: building naive baseline for comparison")
+    if run_docker(["build", "-t", tag, str(context)], capture_output=False).returncode != 0:
+        typer.secho(
+            f"warning: {resolution.name}: baseline build failed, no comparison",
+            fg=typer.colors.YELLOW,
+            err=True,
+        )
+        return
+
+    pruned, naive = image_size(resolution.tag), image_size(tag)
+    if not naive:
+        return
+    saved = naive - pruned
+    typer.echo(
+        f"{resolution.name}: pruned {human_size(pruned)} vs naive {human_size(naive)} "
+        f"compressed  saved {human_size(saved)} ({saved / naive:.0%})"
+    )
+
+
+@app.command()
+def build(
+    service: Annotated[
+        list[str] | None, typer.Argument(help="Services to build. Default: all stale.")
+    ] = None,
+    repo: RepoOption = Path("."),
+    include: IncludeOption = None,
+    show_dockerfile: Annotated[
+        bool, typer.Option("--show-dockerfile", help="Write the context and stop.")
+    ] = False,
+    force: Annotated[bool, typer.Option("--force", help="Rebuild even when unchanged.")] = False,
+    compare: Annotated[
+        bool, typer.Option("--compare", help="Also build a whole-repo image and show the delta.")
+    ] = False,
+    no_verify: Annotated[
+        bool, typer.Option("--no-verify", help="Skip container verification.")
+    ] = False,
+    no_shared_base: Annotated[
+        bool, typer.Option("--no-shared-base", help="Produce self-contained images.")
+    ] = False,
+) -> None:
+    """Build minimal images for the named services, or every stale service."""
+    repo = repo.resolve()
+    includes = [(repo / path).resolve() for path in (include or [])]
+    for path in includes:
+        if not path.exists():
+            _fail(f"--include path does not exist: {path}")
+
+    try:
+        config = load_config(repo)
+    except PyhuskError as exc:
+        _fail(str(exc))
+
+    resolutions = _resolve_all(repo, _selected(config, service), includes, not no_shared_base)
+    _print_warnings(resolutions)
+
+    for name, resolution in sorted(resolutions.items()):
+        assemble_context(
+            repo,
+            name,
+            resolution.service,
+            resolution.files,
+            resolution.requirements.text,
+            resolution.dockerfile_text,
+        )
+
+    if show_dockerfile:
+        for name in sorted(resolutions):
+            typer.echo(f"{name}: wrote {(repo / '.build' / name).relative_to(repo)}/")
+        return
+
+    try:
+        require_docker()
+    except PyhuskError as exc:
+        _fail(str(exc))
+
+    pending = {
+        name: resolution
+        for name, resolution in resolutions.items()
+        if force or is_stale(resolution)
+    }
+    for name in sorted(set(resolutions) - set(pending)):
+        typer.echo(f"{name}: unchanged")
+    if not pending:
+        return
+
+    _build_bases(repo, pending)
+
+    total_repo_files = count_python_files(repo)
+    failures: list[str] = []
+    for name, resolution in sorted(pending.items()):
+        typer.echo(f"{name}: building")
+        built = run_docker(
+            [
+                "build",
+                "-t",
+                resolution.tag,
+                "--label",
+                f"{SLICE_LABEL}={resolution.slice_hash}",
+                str(repo / ".build" / name),
+            ],
+            capture_output=False,
+        )
+        if built.returncode != 0:
+            typer.secho(f"{name}: docker build failed", fg=typer.colors.RED, err=True)
+            failures.append(name)
+            continue
+
+        if not no_verify:
+            outcome = verify(
+                resolution.tag, resolution.service, resolution.routes, resolution.openapi_url
+            )
+            if not outcome.ok:
+                typer.secho(
+                    f"{name}: verification failed ({outcome.tier})\n{outcome.detail}",
+                    fg=typer.colors.RED,
+                    err=True,
+                )
+                # The image is already stamped with this content's hash. Left in
+                # place, the next build would compare hashes, see a match, and
+                # report "unchanged" - a broken image would never be retried.
+                run_docker(["rmi", "-f", resolution.tag])
+                typer.secho(
+                    f"{name}: removed {resolution.tag}; the next build will retry",
+                    fg=typer.colors.RED,
+                    err=True,
+                )
+                failures.append(name)
+                continue
+            if outcome.tier != "schema":
+                typer.secho(
+                    f"warning: {name}: degraded verification ({outcome.tier}) - {outcome.detail}",
+                    fg=typer.colors.YELLOW,
+                    err=True,
+                )
+
+        if resolution.remote_tag:
+            run_docker(["tag", resolution.tag, resolution.remote_tag])
+
+        typer.echo(
+            f"{name}: built {resolution.tag}  "
+            f"{len(resolution.files)}/{total_repo_files} files  "
+            f"{len(resolution.requirements.pins)} packages  "
+            f"{human_size(image_size(resolution.tag))} compressed"
+        )
+        if compare:
+            _compare_against_naive(repo, resolution)
+
+    if failures:
+        _fail(f"failed: {', '.join(sorted(failures))}")
+
+
+@app.command()
+def plan(
+    service: Annotated[
+        list[str] | None, typer.Argument(help="Limit to these services.")
+    ] = None,
+    repo: RepoOption = Path("."),
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Machine-readable output for CI.")
+    ] = False,
+    include: IncludeOption = None,
+    no_shared_base: Annotated[bool, typer.Option("--no-shared-base")] = False,
+) -> None:
+    """Report which services are stale, without building anything."""
+    repo = repo.resolve()
+    includes = [(repo / path).resolve() for path in (include or [])]
+    try:
+        config = load_config(repo)
+    except PyhuskError as exc:
+        _fail(str(exc))
+
+    resolutions = _resolve_all(repo, _selected(config, service), includes, not no_shared_base)
+    # Warnings go to stderr so `--json` stdout stays a clean document; a CI job
+    # pipes this straight into a matrix definition.
+    _print_warnings(resolutions)
+
+    entries: list[dict[str, object]] = []
+    bases: dict[str, int] = {}
+    for name, resolution in sorted(resolutions.items()):
+        stale = is_stale(resolution)
+        entries.append(
+            {
+                "name": name,
+                "tag": resolution.tag,
+                "base": resolution.base_tag,
+                "hash": resolution.slice_hash,
+                "stale": stale,
+            }
+        )
+        if resolution.base_tag and stale:
+            bases[resolution.base_tag] = len(resolution.requirements.pins)
+
+    if as_json:
+        typer.echo(
+            json.dumps(
+                {
+                    "bases": [
+                        {"tag": tag, "packages": count} for tag, count in sorted(bases.items())
+                    ],
+                    "services": entries,
+                }
+            )
+        )
+        return
+
+    for entry in entries:
+        typer.echo(f"{entry['name']}\t{'stale' if entry['stale'] else 'unchanged'}")