harnessweaver-cli 0.7.1__py3-none-any.whl

harnessweaver_cli/__init__.py

@@ -0,0 +1,7 @@
+"""Developer tooling for HarnessWeaver: the ``harnessweaver`` command."""
+
+from importlib.metadata import version as _version
+
+__version__: str = _version("harnessweaver-cli")
+
+__all__ = ["__version__"]

harnessweaver_cli/_load.py

@@ -0,0 +1,119 @@
+"""Finding and loading specs for the CLI, including ``git:<ref>:<path>`` sources."""
+
+import io
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+
+import typer
+
+from harnessweaver import Harness
+from harnessweaver.core.errors import HarnessError, SpecError
+from harnessweaver.core.ports import StateStore
+from harnessweaver.spec import EffectiveSpec, load_spec
+
+__all__ = ["fail", "load_effective", "load_harness", "spec_path"]
+
+DEFAULT_SPEC = Path("harness.yaml")
+
+
+def fail(error: HarnessError | str, code: int = 1) -> typer.Exit:
+    """Print an error the way every command does and return the exit to raise.
+
+    Commands that report checks (``eval run``, ``policy test``, ``replay``) exit 1
+    when a check fails and 2 when they can't run, so CI can tell the two apart.
+    """
+    message = error.message if isinstance(error, HarnessError) else error
+    typer.echo(f"error: {message}", err=True)
+    return typer.Exit(code=code)
+
+
+def spec_path(path: Path | None, code: int = 1) -> Path:
+    """The spec to load. Its directory becomes importable, as when the app runs from
+    there, so ``tasks.*.input`` and handler paths resolve."""
+    found = path or DEFAULT_SPEC
+    if not found.is_file():
+        raise fail(f"no spec at {found}; pass the path to harness.yaml", code)
+    directory = str(found.resolve().parent)
+    if directory not in sys.path:
+        sys.path.insert(0, directory)
+    return found
+
+
+def load_harness(
+    path: Path | None,
+    env: str | None,
+    profile_dirs: list[Path],
+    *,
+    state: StateStore | None = None,
+    code: int = 1,
+) -> Harness:
+    """Load and build the harness, the same way an app does (every check runs).
+    ``state`` replaces the spec's state store (evals keep their runs in memory)."""
+    try:
+        effective = load_spec(spec_path(path, code), env=env, profile_dirs=profile_dirs)
+        return Harness.from_spec(effective, state=state)
+    except HarnessError as exc:
+        raise fail(exc, code) from None
+
+
+def _git(*args: str) -> bytes | None:
+    """Run git and return its output, or ``None`` if it fails or isn't installed."""
+    try:
+        done = subprocess.run(["git", *args], capture_output=True, check=False)
+    except OSError:
+        return None
+    return done.stdout if done.returncode == 0 else None
+
+
+def _load_from_git(ref: str, raw_path: str, env: str | None, dirs: list[Path]) -> EffectiveSpec:
+    """Load a spec as it stands at ``ref``, with the profiles that were committed with it.
+
+    The whole tree at the ref is extracted, so ``.harness/profiles`` and file references come
+    from the ref and not the working tree: otherwise a pull request that weakens only a
+    profile would show no difference. A ``--profile-dir`` inside the repository is read from
+    the ref too. One outside it, like ``HARNESSWEAVER_PROFILE_PATH``, isn't in git, so both
+    sides of a diff see the same files.
+
+    Raises:
+        SpecError: If the ref or the path can't be read.
+    """
+    missing = SpecError(f"can't read {raw_path} at {ref} from git")
+    # --end-of-options keeps a ref from being read as an option of git's.
+    commit = _git("rev-parse", "--verify", "--quiet", "--end-of-options", f"{ref}^{{commit}}")
+    top = _git("rev-parse", "--show-toplevel")
+    if commit is None or top is None:
+        raise missing
+    root = Path(top.decode().strip()).resolve()
+    archive = _git("-C", str(root), "archive", "--format=tar", commit.decode().strip())
+    if archive is None:
+        raise missing
+
+    def inside(tree: Path, path: Path) -> Path | None:
+        """Where ``path`` is in the extracted tree, if it's in the repository at all."""
+        resolved = path.resolve()
+        return tree / resolved.relative_to(root) if resolved.is_relative_to(root) else None
+
+    with tempfile.TemporaryDirectory(prefix="harnessweaver-git-") as temporary:
+        tree = Path(temporary).resolve()
+        with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
+            tar.extractall(tree, filter="data")  # the data filter refuses paths out of the tree
+        path = inside(tree, Path(raw_path))
+        if path is None or not path.is_file():
+            raise missing
+        return load_spec(path, env=env, profile_dirs=[inside(tree, d) or d for d in dirs])
+
+
+def load_effective(source: str, env: str | None, profile_dirs: list[Path]) -> EffectiveSpec:
+    """Load a spec from a file, or from git as ``git:<ref>:<path>`` (for ``spec diff`` in CI)."""
+    try:
+        if not source.startswith("git:"):
+            return load_spec(spec_path(Path(source)), env=env, profile_dirs=profile_dirs)
+        ref, _, raw_path = source.removeprefix("git:").partition(":")
+        if not ref or not raw_path:
+            raise SpecError(f"{source!r} must look like git:<ref>:<path>")
+        return _load_from_git(ref, raw_path, env, profile_dirs)
+    except HarnessError as exc:
+        raise fail(exc) from None

harnessweaver_cli/_yaml.py

@@ -0,0 +1,49 @@
+"""Comment-preserving reads and writes of the YAML files the CLI edits (ruamel.yaml, ADR-0016).
+
+The spec loader reads YAML 1.2 with PyYAML (ADR-0017). Editing commands
+(``add guard``, ``add tool``, ``tools pin``) use ruamel.yaml's round-trip mode
+instead, so the comments, key order and flow style a person wrote survive the
+edit. ruamel.yaml reads YAML 1.2 too, so both see the same values.
+"""
+
+import io
+from pathlib import Path
+from typing import Any
+
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+
+__all__ = ["dump", "flow", "read", "write"]
+
+
+def _yaml() -> YAML:
+    yaml = YAML()  # round-trip mode: comments, order and quoting are kept
+    yaml.preserve_quotes = True
+    yaml.width = 4096  # never re-wrap a long line someone wrote on purpose
+    # The repository's style: list items indented two spaces under their key.
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    return yaml
+
+
+def read(path: Path) -> Any:
+    """The document at ``path``, as ruamel's comment-keeping maps and lists."""
+    return _yaml().load(path.read_text(encoding="utf-8"))
+
+
+def dump(document: Any) -> str:
+    """``document`` as YAML text, with its comments."""
+    out = io.StringIO()
+    _yaml().dump(document, out)
+    return out.getvalue()
+
+
+def write(path: Path, document: Any) -> None:
+    """Write ``document`` to ``path``, comments included."""
+    path.write_text(dump(document), encoding="utf-8")
+
+
+def flow(**items: Any) -> CommentedMap:
+    """A map written on one line, ``{ id: x, use: y }``, as spec list items usually are."""
+    found = CommentedMap(items)
+    found.fa.set_flow_style()
+    return found

harnessweaver_cli/add.py

@@ -0,0 +1,238 @@
+"""``harnessweaver add guard`` and ``harnessweaver add tool``: grow a spec without hand-editing it.
+
+Both edit files in place with comments kept (``_yaml``), then load the spec the
+way the app would. A guard whose edit doesn't load is taken back out, so a
+failed ``add`` leaves the spec as it was.
+"""
+
+import inspect
+import re
+import typing
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+import typer
+from pydantic import Field, ValidationError, create_model
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
+from harnessweaver import Harness, Risk, SideEffect, ToolManifest
+from harnessweaver.core.errors import HarnessError
+from harnessweaver.core.jsonschema import SchemaError, check_schema
+from harnessweaver.spec import load_spec
+from harnessweaver.spec.yaml import load_yaml
+from harnessweaver_cli import _yaml
+from harnessweaver_cli._load import fail, spec_path
+from harnessweaver_cli.spec import Env, Path_, ProfileDirs
+
+__all__ = ["add_app", "input_schema"]
+
+add_app = typer.Typer(help="Add a guard or a tool to the spec.", no_args_is_help=True)
+
+Stage = Literal["input", "context", "tool_result", "output", "stream"]
+
+_PARAM = re.compile(r"^(\w+)\s*(?:\([^)]*\))?\s*:\s*(.*)$")
+"""A docstring ``Args:`` entry: ``name: text`` or ``name (type): text``."""
+
+_IDEMPOTENCY_KEY = "idempotency_key"
+"""A native tool's parameter the governor fills in; the model never sees it (DESIGNS No. 17)."""
+
+
+def _settings(items: list[str]) -> dict[str, Any]:
+    """``--set KEY=VALUE`` options as provider settings, each value read as YAML."""
+    found: dict[str, Any] = {}
+    for item in items:
+        key, sep, raw = item.partition("=")
+        if not sep or not key:
+            raise fail(f"--set {item!r} must look like KEY=VALUE")
+        found[key] = load_yaml(raw, f"--set {key}") if raw else ""
+    return found
+
+
+def _check(path: Path, env: str | None, profile_dirs: list[Path]) -> None:
+    """Load and build the harness from ``path``, as ``spec validate`` does."""
+    Harness.from_spec(load_spec(path, env=env, profile_dirs=profile_dirs))
+
+
+@add_app.command("guard")
+def add_guard(
+    use: Annotated[str, typer.Argument(help="The guard provider, such as pii.regex or acme.pii.")],
+    path: Path_ = None,
+    stage: Annotated[Stage, typer.Option("--stage", help="Where it checks content.")] = "input",
+    guard_id: Annotated[
+        str | None, typer.Option("--id", help="Its id (default: the provider name with dashes).")
+    ] = None,
+    action: Annotated[
+        str | None, typer.Option("--action", help="What it does when it fires (overrides its own).")
+    ] = None,
+    settings: Annotated[
+        list[str] | None, typer.Option("--set", help="A provider setting, KEY=VALUE; repeatable.")
+    ] = None,
+    env: Env = None,
+    profile_dir: ProfileDirs = None,
+) -> None:
+    """Add a guard in shadow mode (``mode: log_only``): it records what it would do and
+    changes nothing until a reviewed edit promotes it to ``enforce`` (R8)."""
+    target = spec_path(path)
+    original = target.read_text(encoding="utf-8")
+    document = _yaml.read(target)
+    if not isinstance(document, CommentedMap):
+        raise fail(f"{target} must be a YAML map")
+    name = guard_id or use.replace(".", "-")
+    guards = document.setdefault("guardrails", CommentedMap()).setdefault(stage, CommentedSeq())
+    if any(isinstance(g, dict) and g.get("id") == name for g in guards):
+        raise fail(f"guardrails.{stage} already has a guard {name!r}; pick another --id")
+    item = _yaml.flow(id=name, use=use, mode="log_only")
+    if action is not None:
+        item["action"] = action
+    item.update(_settings(settings or []))
+    guards.append(item)
+    _yaml.write(target, document)
+    try:
+        _check(target, env, profile_dir or [])
+    except HarnessError as exc:
+        target.write_text(original, encoding="utf-8")  # leave the spec as it was
+        raise fail(exc) from None
+    typer.echo(f"added guardrails.{stage}[{name}] ({use}) in shadow mode (mode: log_only)")
+    typer.echo("  promote it with mode: enforce once the eval shadow report backs it (R8)")
+
+
+# --- add tool ----------------------------------------------------------------------------------
+
+
+def _docs(function: Any) -> tuple[str, dict[str, str]]:
+    """A function's summary and its parameter descriptions, from a Google-style docstring.
+
+    The summary is the text before the first section header (``Args:``,
+    ``Returns:``, ...). Parameter descriptions are the ``name: text`` lines under
+    ``Args:``, with indented continuation lines joined on.
+    """
+    lines = (inspect.getdoc(function) or "").splitlines()
+    summary: list[str] = []
+    params: dict[str, str] = {}
+    section: str | None = None
+    current: str | None = None
+    for line in lines:
+        stripped = line.strip()
+        if stripped.endswith(":") and stripped[:-1] in ("Args", "Arguments", "Returns", "Raises"):
+            section, current = stripped[:-1], None
+            continue
+        if section is None:
+            summary.append(stripped)
+        elif section in ("Args", "Arguments") and stripped:
+            # Entries sit one indent under "Args:"; deeper lines continue the last one.
+            entry = None if line.startswith("        ") else _PARAM.match(stripped)
+            if entry is not None:
+                current = str(entry[1])
+                params[current] = str(entry[2]).strip()
+            elif current is not None:
+                params[current] = f"{params[current]} {stripped}".strip()
+    return " ".join(s for s in summary if s).strip(), params
+
+
+def input_schema(function: Any) -> dict[str, Any]:
+    """The JSON Schema of a function's parameters, as a tool manifest's ``input_schema``.
+
+    Pydantic builds it from the type hints and defaults; ``idempotency_key``
+    is left out because the governor supplies it. Titles are dropped, since
+    only the property names and descriptions mean anything to a model.
+
+    Raises:
+        typer.Exit: For ``*args`` or ``**kwargs``, which a schema can't describe.
+    """
+    _, described = _docs(function)
+    hints = typing.get_type_hints(function)
+    fields: dict[str, Any] = {}
+    for name, parameter in inspect.signature(function).parameters.items():
+        if name == _IDEMPOTENCY_KEY:
+            continue
+        if parameter.kind in (parameter.VAR_POSITIONAL, parameter.VAR_KEYWORD):
+            raise fail(f"{function.__name__}: *args and **kwargs can't be described as a tool")
+        default = ... if parameter.default is parameter.empty else parameter.default
+        fields[name] = (hints.get(name, Any), Field(default, description=described.get(name)))
+    schema = create_model(f"{function.__name__}_args", **fields).model_json_schema()
+    schema.pop("title", None)
+    for prop in schema.get("properties", {}).values():
+        prop.pop("title", None)
+    return schema
+
+
+def _manifest_globs(document: CommentedMap) -> CommentedSeq:
+    """The spec's ``tools.manifests`` as a list, made one if it was a single glob or missing."""
+    tools = document.setdefault("tools", CommentedMap())
+    globs = tools.get("manifests")
+    if isinstance(globs, CommentedSeq):
+        return globs
+    found = CommentedSeq([globs] if isinstance(globs, str) else list(globs or []))
+    tools["manifests"] = found
+    return found
+
+
+@add_app.command("tool")
+def add_tool(
+    handler: Annotated[str, typer.Argument(help="The tool's function, 'package.module:function'.")],
+    path: Path_ = None,
+    directory: Annotated[
+        Path, typer.Option("--dir", help="Where manifests live, relative to the spec.")
+    ] = Path("tools"),
+    risk: Annotated[Risk, typer.Option("--risk", help="How much harm a misused call does.")] = (
+        Risk.LOW
+    ),
+    side_effect: Annotated[
+        SideEffect, typer.Option("--side-effect", help="What a call changes.")
+    ] = SideEffect.NONE,
+    force: Annotated[bool, typer.Option("--force", help="Overwrite an existing manifest.")] = False,
+) -> None:
+    """Write a tool manifest from a function's signature and docstring (S2).
+
+    The name is the function's, the description its docstring summary, and the
+    input schema comes from its parameters. Review it, then pin it with
+    ``harnessweaver tools pin``.
+    """
+    target = spec_path(path)
+    module, _, attribute = handler.partition(":")
+    if not module or not attribute:
+        raise fail(f"{handler!r} isn't 'package.module:function'")
+    try:
+        function = getattr(__import__(module, fromlist=[attribute]), attribute)
+    except (ImportError, AttributeError) as exc:
+        raise fail(f"can't import {handler!r} ({type(exc).__name__})") from None
+    if not callable(function):
+        raise fail(f"{handler!r} isn't a function")
+    description, _ = _docs(function)
+    if not description:
+        raise fail(f"{handler} has no docstring; the model needs a description of the tool")
+    name = function.__name__
+    manifest = {
+        "apiVersion": "harnessweaver/v1",
+        "kind": "Tool",
+        "metadata": {"name": name, "version": "1.0.0"},
+        "spec": {
+            "transport": "native",
+            "handler": handler,
+            "description": description,
+            "input_schema": input_schema(function),
+            "side_effect": side_effect.value,
+            "risk": risk.value,
+        },
+    }
+    try:
+        check_schema(ToolManifest.model_validate(manifest).spec.input_schema)
+    except (ValidationError, SchemaError) as exc:
+        raise fail(f"{handler}: its manifest isn't valid: {exc}") from None
+    base = target.resolve().parent
+    file = base / directory / f"{name}.tool.yaml"
+    if file.exists() and not force:
+        raise fail(f"{file} already exists; use --force to overwrite")
+    file.parent.mkdir(parents=True, exist_ok=True)
+    header = f"# {name}: written by harnessweaver add tool. Review it, then pin it (tools pin).\n"
+    file.write_text(header + _yaml.dump(manifest), encoding="utf-8")
+    typer.echo(f"wrote {file.relative_to(base)}")
+    # Make sure the spec reads the new manifest; add the directory's glob if it doesn't.
+    document = _yaml.read(target)
+    globs = _manifest_globs(document)
+    relative = file.relative_to(base)
+    if not any(relative.match(str(g)) for g in globs):
+        globs.append(f"{directory.as_posix()}/*.tool.yaml")
+        _yaml.write(target, document)
+        typer.echo(f"  added {directory.as_posix()}/*.tool.yaml to tools.manifests")
+    typer.echo("  next: review the description and schema, then run harnessweaver tools pin")

harnessweaver_cli/dev.py

@@ -0,0 +1,141 @@
+"""``harnessweaver dev``: a local stack for the app, pre-wired to its spec (ADR-0039).
+
+It writes ``.harness/dev/`` next to the spec: the LiteLLM config ``gateway
+litellm`` generates, an OpenTelemetry collector config, a Docker Compose file
+with LiteLLM, the collector, Langfuse and harnessd, and a ``.env`` of local
+secrets. The ``.env`` is written once and then kept, so its secrets and the
+values the developer fills in survive the next run; the other files are
+regenerated each time. ``--up`` starts the stack.
+"""
+
+import base64
+import secrets
+import shutil
+import subprocess
+from importlib import resources
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from harnessweaver.core.errors import HarnessError
+from harnessweaver.spec import EffectiveSpec, load_spec
+from harnessweaver_cli._load import fail, spec_path
+from harnessweaver_cli.gateway import gateway_tools, litellm_config, render_litellm, upstream_env
+from harnessweaver_cli.spec import Env, Path_, ProfileDirs
+
+__all__ = ["compose_file", "dev", "env_file"]
+
+PROVIDER_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY")
+"""Provider keys LiteLLM reads from its environment; only the gateway holds them (S1)."""
+
+
+def _token(prefix: str = "") -> str:
+    return prefix + secrets.token_hex(16)
+
+
+def env_file(effective: EffectiveSpec) -> str:
+    """A ``.env`` for the stack: random local secrets, and empty lines for what only
+    the developer knows (the provider model behind each gateway model, and the keys).
+
+    The app's own key variable (``models.api_key_env``) gets LiteLLM's master key,
+    so the app, harnessd and the gateway agree without further setup.
+    """
+    names = [
+        entry["model_name"]
+        for entry in litellm_config(effective, gateway_tools(effective))["model_list"]
+    ]
+    master = _token("sk-dev-")
+    public, secret = _token("pk-lf-"), _token("sk-lf-")
+    app_key = effective.spec.models.provider_settings().get("api_key_env", "LITELLM_API_KEY")
+    lines = [
+        "# Local secrets for `harnessweaver dev`, generated once. Never commit this file.",
+        "#",
+        "# The provider model behind each gateway model, as LiteLLM names it (provider/model):",
+        *[f"{upstream_env(name)}=" for name in names],
+        "# Provider keys, which only the gateway holds (S1):",
+        *[f"{key}=" for key in PROVIDER_KEYS],
+        "",
+        f"LITELLM_MASTER_KEY={master}",
+        f"{app_key}={master}",
+        f"HARNESSD_TOKEN={_token()}",
+        f"POSTGRES_PASSWORD={_token()}",
+        f"CLICKHOUSE_PASSWORD={_token()}",
+        f"REDIS_AUTH={_token()}",
+        f"MINIO_ROOT_PASSWORD={_token()}",
+        f"SALT={_token()}",
+        f"ENCRYPTION_KEY={secrets.token_hex(32)}",
+        f"NEXTAUTH_SECRET={_token()}",
+        f"LANGFUSE_PUBLIC_KEY={public}",
+        f"LANGFUSE_SECRET_KEY={secret}",
+        f"LANGFUSE_OTLP_AUTH={base64.b64encode(f'{public}:{secret}'.encode()).decode()}",
+        "LANGFUSE_USER_EMAIL=dev@example.com",
+        f"LANGFUSE_USER_PASSWORD={_token()}",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def compose_file(spec: Path, env: str | None, profile_dirs: list[Path]) -> str:
+    """The Compose file, for this spec and overlay. Each ``--profile-dir`` folder is
+    mounted read-only into harnessd's container and put on its profile path, so it
+    loads the spec as the developer's machine does."""
+    mounts = [f"/profiles/{index}" for index in range(len(profile_dirs))]
+    volumes = "".join(
+        f"      - {folder.resolve()}:{mount}:ro\n"
+        for folder, mount in zip(profile_dirs, mounts, strict=True)
+    )
+    return (
+        _template("compose.yaml")
+        .replace("__spec__", spec.name)
+        .replace("__env__", env or "")
+        .replace("__profile_path__", ":".join(mounts))
+        .replace("__profile_volumes__", volumes)
+    )
+
+
+def _template(name: str) -> str:
+    return (
+        resources.files("harnessweaver_cli")
+        .joinpath("templates", "_dev", name)
+        .read_text(encoding="utf-8")
+    )
+
+
+def dev(
+    path: Path_ = None,
+    env: Env = None,
+    profile_dir: ProfileDirs = None,
+    up: Annotated[bool, typer.Option("--up", help="Start it: docker compose up -d.")] = False,
+) -> None:
+    """Write a local stack: LiteLLM (from the spec), an OTel collector, Langfuse and harnessd."""
+    found = spec_path(path)
+    try:
+        effective = load_spec(found, env=env, profile_dirs=profile_dir or [])
+        litellm = render_litellm(effective)
+    except HarnessError as exc:
+        raise fail(exc) from None
+    directory = found.resolve().parent / ".harness" / "dev"
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / ".gitignore").write_text("*\n", encoding="utf-8")  # nothing here is committed
+    (directory / "litellm.yaml").write_text(litellm, encoding="utf-8")
+    (directory / "otel-collector.yaml").write_text(_template("otel-collector.yaml"), "utf-8")
+    (directory / "compose.yaml").write_text(compose_file(found, env, profile_dir or []), "utf-8")
+    secrets_file = directory / ".env"
+    if not secrets_file.exists():
+        secrets_file.write_text(env_file(effective), encoding="utf-8")
+        typer.echo(f"wrote {secrets_file}: fill in the provider models and keys")
+    typer.echo(f"wrote the dev stack to {directory}")
+    if not up:
+        typer.echo(
+            f"start it with: harnessweaver dev --up  (or docker compose up -d in {directory})"
+        )
+        return
+    docker = shutil.which("docker")
+    if docker is None:
+        raise fail("docker isn't installed; install Docker to start the stack")
+    done = subprocess.run([docker, "compose", "up", "-d"], cwd=directory, check=False)
+    if done.returncode != 0:
+        raise typer.Exit(code=done.returncode)
+    typer.echo(
+        "LiteLLM http://127.0.0.1:4000, harnessd http://127.0.0.1:7070, Langfuse http://127.0.0.1:3000"
+    )