harnessweaver-cli 0.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harnessweaver_cli/__init__.py +7 -0
- harnessweaver_cli/_load.py +119 -0
- harnessweaver_cli/_yaml.py +49 -0
- harnessweaver_cli/add.py +238 -0
- harnessweaver_cli/dev.py +141 -0
- harnessweaver_cli/doctor.py +185 -0
- harnessweaver_cli/evals.py +276 -0
- harnessweaver_cli/gateway.py +301 -0
- harnessweaver_cli/main.py +62 -0
- harnessweaver_cli/plugins.py +105 -0
- harnessweaver_cli/policy.py +143 -0
- harnessweaver_cli/py.typed +0 -0
- harnessweaver_cli/replay.py +75 -0
- harnessweaver_cli/scaffold.py +158 -0
- harnessweaver_cli/sidecar.py +48 -0
- harnessweaver_cli/spec.py +160 -0
- harnessweaver_cli/templates/_dev/compose.yaml +183 -0
- harnessweaver_cli/templates/_dev/otel-collector.yaml +25 -0
- harnessweaver_cli/templates/_plugin/README.md +26 -0
- harnessweaver_cli/templates/_plugin/pyproject.toml +26 -0
- harnessweaver_cli/templates/_plugin/src/__ns___harness/__init__.py +20 -0
- harnessweaver_cli/templates/_plugin/src/__ns___harness/guards.py +48 -0
- harnessweaver_cli/templates/_plugin/src/__ns___harness/py.typed +0 -0
- harnessweaver_cli/templates/_plugin/tests/test_guards.py +26 -0
- harnessweaver_cli/templates/agent/README.md +18 -0
- harnessweaver_cli/templates/agent/app/__init__.py +1 -0
- harnessweaver_cli/templates/agent/app/agent.py +12 -0
- harnessweaver_cli/templates/agent/app/main.py +19 -0
- harnessweaver_cli/templates/agent/app/tools.py +28 -0
- harnessweaver_cli/templates/agent/evals/redteam.yaml +17 -0
- harnessweaver_cli/templates/agent/harness.offline.yaml +16 -0
- harnessweaver_cli/templates/agent/harness.yaml +38 -0
- harnessweaver_cli/templates/agent/pyproject.toml +13 -0
- harnessweaver_cli/templates/agent/tests/test_agent.py +54 -0
- harnessweaver_cli/templates/agent/tools/lookup_order.tool.yaml +15 -0
- harnessweaver_cli/templates/agent/tools/refund_payment.tool.yaml +17 -0
- harnessweaver_cli/templates/chat/README.md +15 -0
- harnessweaver_cli/templates/chat/app/__init__.py +1 -0
- harnessweaver_cli/templates/chat/app/chat.py +32 -0
- harnessweaver_cli/templates/chat/app/main.py +33 -0
- harnessweaver_cli/templates/chat/evals/golden.yaml +10 -0
- harnessweaver_cli/templates/chat/harness.offline.yaml +12 -0
- harnessweaver_cli/templates/chat/harness.yaml +39 -0
- harnessweaver_cli/templates/chat/pyproject.toml +13 -0
- harnessweaver_cli/templates/chat/tests/test_chat.py +63 -0
- harnessweaver_cli/templates/extraction/README.md +16 -0
- harnessweaver_cli/templates/extraction/app/__init__.py +1 -0
- harnessweaver_cli/templates/extraction/app/main.py +21 -0
- harnessweaver_cli/templates/extraction/app/models.py +20 -0
- harnessweaver_cli/templates/extraction/evals/golden.yaml +12 -0
- harnessweaver_cli/templates/extraction/harness.offline.yaml +18 -0
- harnessweaver_cli/templates/extraction/harness.yaml +33 -0
- harnessweaver_cli/templates/extraction/prompts/extract@1.md +7 -0
- harnessweaver_cli/templates/extraction/pyproject.toml +13 -0
- harnessweaver_cli/templates/extraction/tests/test_extract.py +56 -0
- harnessweaver_cli/templates/rag/README.md +17 -0
- harnessweaver_cli/templates/rag/app/__init__.py +1 -0
- harnessweaver_cli/templates/rag/app/flow.py +29 -0
- harnessweaver_cli/templates/rag/app/main.py +19 -0
- harnessweaver_cli/templates/rag/data/kb.yaml +13 -0
- harnessweaver_cli/templates/rag/evals/golden.yaml +14 -0
- harnessweaver_cli/templates/rag/harness.offline.yaml +13 -0
- harnessweaver_cli/templates/rag/harness.yaml +50 -0
- harnessweaver_cli/templates/rag/prompts/answer@1.md +9 -0
- harnessweaver_cli/templates/rag/pyproject.toml +13 -0
- harnessweaver_cli/templates/rag/tests/test_answer.py +50 -0
- harnessweaver_cli/templates/workflow/README.md +16 -0
- harnessweaver_cli/templates/workflow/app/__init__.py +1 -0
- harnessweaver_cli/templates/workflow/app/main.py +20 -0
- harnessweaver_cli/templates/workflow/app/steps.py +49 -0
- harnessweaver_cli/templates/workflow/evals/golden.yaml +9 -0
- harnessweaver_cli/templates/workflow/harness.offline.yaml +12 -0
- harnessweaver_cli/templates/workflow/harness.yaml +39 -0
- harnessweaver_cli/templates/workflow/prompts/classify@1.md +4 -0
- harnessweaver_cli/templates/workflow/prompts/reply@1.md +5 -0
- harnessweaver_cli/templates/workflow/prompts/summarize@1.md +3 -0
- harnessweaver_cli/templates/workflow/pyproject.toml +13 -0
- harnessweaver_cli/templates/workflow/tests/test_pipeline.py +32 -0
- harnessweaver_cli/tools.py +72 -0
- harnessweaver_cli-0.7.1.dist-info/METADATA +83 -0
- harnessweaver_cli-0.7.1.dist-info/RECORD +84 -0
- harnessweaver_cli-0.7.1.dist-info/WHEEL +4 -0
- harnessweaver_cli-0.7.1.dist-info/entry_points.txt +2 -0
- harnessweaver_cli-0.7.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""``harnessweaver init``: start a project from a template, and ``harnessweaver test``.
|
|
2
|
+
|
|
3
|
+
Templates are directories copied as they are; the only substitution is the
|
|
4
|
+
project name for the marker ``__app_name__`` in file contents (there is no
|
|
5
|
+
template language, R12). Built-in templates ship with the CLI; a plugin's
|
|
6
|
+
templates are named ``<namespace>.<template>`` (DESIGNS No. 13). Templates whose
|
|
7
|
+
names start with ``_`` are scaffolds for other commands (``plugin new``), not
|
|
8
|
+
projects.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from importlib import resources
|
|
17
|
+
from importlib.resources.abc import Traversable
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Annotated, Final
|
|
20
|
+
|
|
21
|
+
import typer
|
|
22
|
+
|
|
23
|
+
from harnessweaver.core.errors import HarnessError
|
|
24
|
+
from harnessweaver.ports import TEST_MODE_ENV
|
|
25
|
+
from harnessweaver_cli._load import fail
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"BUILTIN_TEMPLATES",
|
|
29
|
+
"copy_tree",
|
|
30
|
+
"init",
|
|
31
|
+
"render_template",
|
|
32
|
+
"run_tests",
|
|
33
|
+
"template_root",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
MARKER: Final = "__app_name__"
|
|
37
|
+
BUILTIN_TEMPLATES: Final = ("extraction", "rag", "agent", "workflow", "chat")
|
|
38
|
+
_NAME = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
39
|
+
_SKIP = {"__pycache__", ".pytest_cache", ".DS_Store"}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def template_root(name: str) -> Traversable:
|
|
43
|
+
"""Where template ``name`` lives: in the CLI, or in the plugin that ships it.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
typer.Exit: For a template nobody ships.
|
|
47
|
+
"""
|
|
48
|
+
if name.startswith("_"):
|
|
49
|
+
raise fail(f"no template {name!r}; built-in templates: {', '.join(BUILTIN_TEMPLATES)}")
|
|
50
|
+
return _root(name)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _root(name: str) -> Traversable:
|
|
54
|
+
if "." not in name:
|
|
55
|
+
root = resources.files("harnessweaver_cli").joinpath("templates", name)
|
|
56
|
+
if not root.is_dir():
|
|
57
|
+
raise fail(f"no template {name!r}; built-in templates: {', '.join(BUILTIN_TEMPLATES)}")
|
|
58
|
+
return root
|
|
59
|
+
from harnessweaver.core.registry import PluginCatalog
|
|
60
|
+
|
|
61
|
+
namespace, _, template = name.partition(".")
|
|
62
|
+
try:
|
|
63
|
+
loaded = PluginCatalog.default().load(namespace)
|
|
64
|
+
except HarnessError as exc:
|
|
65
|
+
raise fail(exc) from None
|
|
66
|
+
path = loaded.plugin.templates.get(template)
|
|
67
|
+
if path is None:
|
|
68
|
+
raise fail(f"plugin {namespace!r} has no template {template!r}")
|
|
69
|
+
return loaded.resource(path)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _files(root: Traversable, prefix: Path = Path()) -> list[tuple[Path, Traversable]]:
|
|
73
|
+
found: list[tuple[Path, Traversable]] = []
|
|
74
|
+
for entry in sorted(root.iterdir(), key=lambda e: e.name):
|
|
75
|
+
if entry.name in _SKIP:
|
|
76
|
+
continue
|
|
77
|
+
if entry.is_dir():
|
|
78
|
+
found += _files(entry, prefix / entry.name)
|
|
79
|
+
else:
|
|
80
|
+
found.append((prefix / entry.name, entry))
|
|
81
|
+
return found
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _filled(text: str, markers: Mapping[str, str]) -> str:
|
|
85
|
+
for marker, value in markers.items():
|
|
86
|
+
text = text.replace(marker, value)
|
|
87
|
+
return text
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def copy_tree(
|
|
91
|
+
root: Traversable, directory: Path, markers: Mapping[str, str], *, force: bool = False
|
|
92
|
+
) -> list[Path]:
|
|
93
|
+
"""Copy the files under ``root`` into ``directory``, replacing each marker with its
|
|
94
|
+
value in file contents and in paths. Returns the files written.
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
typer.Exit: If a file would be overwritten (without ``force``).
|
|
98
|
+
"""
|
|
99
|
+
files = [(Path(_filled(str(relative), markers)), source) for relative, source in _files(root)]
|
|
100
|
+
clashes = [str(relative) for relative, _ in files if (directory / relative).exists()]
|
|
101
|
+
if clashes and not force:
|
|
102
|
+
raise fail(f"{', '.join(clashes)} already exist in {directory}; use --force to overwrite")
|
|
103
|
+
written: list[Path] = []
|
|
104
|
+
for relative, source in files:
|
|
105
|
+
target = directory / relative
|
|
106
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
107
|
+
target.write_text(_filled(source.read_text(encoding="utf-8"), markers), encoding="utf-8")
|
|
108
|
+
written.append(target)
|
|
109
|
+
return written
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def render_template(
|
|
113
|
+
name: str, directory: Path, app_name: str, *, force: bool = False
|
|
114
|
+
) -> list[Path]:
|
|
115
|
+
"""Copy template ``name`` into ``directory``, named ``app_name``. Returns the files written.
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
typer.Exit: For an unknown template, or if a file would be overwritten
|
|
119
|
+
(without ``force``).
|
|
120
|
+
"""
|
|
121
|
+
return copy_tree(template_root(name), directory, {MARKER: app_name}, force=force)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def init(
|
|
125
|
+
directory: Annotated[Path, typer.Argument(help="Where to create the project.")] = Path("."),
|
|
126
|
+
template: Annotated[
|
|
127
|
+
str, typer.Option("--template", "-t", help="A template name.")
|
|
128
|
+
] = "extraction",
|
|
129
|
+
name: Annotated[
|
|
130
|
+
str | None, typer.Option("--name", help="Project name (default: the directory).")
|
|
131
|
+
] = None,
|
|
132
|
+
force: Annotated[bool, typer.Option("--force", help="Overwrite existing files.")] = False,
|
|
133
|
+
) -> None:
|
|
134
|
+
"""Start a project: harness.yaml, an offline overlay, code, prompts, evals and tests."""
|
|
135
|
+
app_name = name or directory.resolve().name.lower().replace("_", "-")
|
|
136
|
+
if not _NAME.match(app_name):
|
|
137
|
+
raise fail(f"invalid project name {app_name!r}: use lowercase letters, digits and dashes")
|
|
138
|
+
written = render_template(template, directory, app_name, force=force)
|
|
139
|
+
typer.echo(f"created {len(written)} files in {directory} from the {template} template")
|
|
140
|
+
typer.echo("next:")
|
|
141
|
+
typer.echo(f" cd {directory}")
|
|
142
|
+
typer.echo(" harnessweaver spec validate --env offline")
|
|
143
|
+
typer.echo(" harnessweaver test")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def run_tests(
|
|
147
|
+
ctx: typer.Context,
|
|
148
|
+
env: Annotated[str, typer.Option("--env", "-e", help="Overlay for the tests.")] = "offline",
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Run the app's tests offline: test mode on, the offline overlay, extra args to pytest."""
|
|
151
|
+
environment = {**os.environ, TEST_MODE_ENV: "1", "HARNESSWEAVER_ENV": env}
|
|
152
|
+
try:
|
|
153
|
+
done = subprocess.run(
|
|
154
|
+
[sys.executable, "-m", "pytest", *ctx.args], env=environment, check=False
|
|
155
|
+
)
|
|
156
|
+
except OSError as exc:
|
|
157
|
+
raise fail(f"can't run pytest ({type(exc).__name__})") from None
|
|
158
|
+
raise typer.Exit(code=done.returncode)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""``harnessweaver sidecar``: run harnessd for a spec, if harnessweaver-sidecar is installed.
|
|
2
|
+
|
|
3
|
+
The CLI doesn't depend on the sidecar package (it brings FastAPI and uvicorn), so
|
|
4
|
+
the import happens only when the command runs.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
from typing import Annotated
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from harnessweaver.core.errors import HarnessError
|
|
13
|
+
from harnessweaver_cli._load import fail, spec_path
|
|
14
|
+
from harnessweaver_cli.spec import Env, Path_, ProfileDirs
|
|
15
|
+
|
|
16
|
+
__all__ = ["sidecar"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sidecar(
|
|
20
|
+
path: Path_ = None,
|
|
21
|
+
env: Env = None,
|
|
22
|
+
profile_dir: ProfileDirs = None,
|
|
23
|
+
host: Annotated[str, typer.Option(help="Where to listen; keep 127.0.0.1 (S6).")] = "127.0.0.1",
|
|
24
|
+
port: Annotated[int, typer.Option(help="The port.")] = 7070,
|
|
25
|
+
token_env: Annotated[
|
|
26
|
+
str, typer.Option(help="The variable holding the bearer token clients send.")
|
|
27
|
+
] = "HARNESSD_TOKEN",
|
|
28
|
+
max_body_bytes: Annotated[
|
|
29
|
+
int, typer.Option(help="The largest request body read; anything over answers 413.")
|
|
30
|
+
] = 10 * 1024 * 1024,
|
|
31
|
+
) -> None:
|
|
32
|
+
"""Serve the spec to other languages over HTTP: run harnessd (DESIGNS No. 24)."""
|
|
33
|
+
try:
|
|
34
|
+
harnessd = importlib.import_module("harnessweaver_sidecar.main")
|
|
35
|
+
except ImportError:
|
|
36
|
+
raise fail("harnessd isn't installed; run: uv add harnessweaver-sidecar") from None
|
|
37
|
+
try:
|
|
38
|
+
harnessd.serve(
|
|
39
|
+
spec_path(path),
|
|
40
|
+
env=env,
|
|
41
|
+
profile_dirs=profile_dir or [],
|
|
42
|
+
host=host,
|
|
43
|
+
port=port,
|
|
44
|
+
token_env=token_env,
|
|
45
|
+
max_body_bytes=max_body_bytes,
|
|
46
|
+
)
|
|
47
|
+
except HarnessError as exc:
|
|
48
|
+
raise fail(exc) from None
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""``harnessweaver spec validate|explain|diff``, ``hooks`` and ``contracts export``."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Annotated, Literal
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from rich import box
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
from rich.markup import escape
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
|
|
13
|
+
from harnessweaver import HookPoint
|
|
14
|
+
from harnessweaver.contracts import stale_schemas, write_schemas
|
|
15
|
+
from harnessweaver.spec import diff_specs, render_diff, render_explain
|
|
16
|
+
from harnessweaver_cli._load import load_effective, load_harness, spec_path
|
|
17
|
+
|
|
18
|
+
__all__ = ["contracts_app", "hooks", "spec_app"]
|
|
19
|
+
|
|
20
|
+
spec_app = typer.Typer(help="Validate, explain and diff harness.yaml.", no_args_is_help=True)
|
|
21
|
+
contracts_app = typer.Typer(help="Work with the language-neutral contracts.", no_args_is_help=True)
|
|
22
|
+
|
|
23
|
+
Path_ = Annotated[Path | None, typer.Argument(help="The app spec (default: harness.yaml).")]
|
|
24
|
+
Env = Annotated[str | None, typer.Option("--env", "-e", help="Overlay: harness.<env>.yaml.")]
|
|
25
|
+
ProfileDirs = Annotated[
|
|
26
|
+
list[Path] | None, typer.Option("--profile-dir", help="Also search here for org:// profiles.")
|
|
27
|
+
]
|
|
28
|
+
Format = Annotated[Literal["text", "json"], typer.Option("--format", help="Output format.")]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _console() -> Console:
|
|
32
|
+
return Console(width=120, highlight=False, soft_wrap=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@spec_app.command("validate")
|
|
36
|
+
def validate(
|
|
37
|
+
path: Path_ = None, env: Env = None, profile_dir: ProfileDirs = None, output: Format = "text"
|
|
38
|
+
) -> None:
|
|
39
|
+
"""Load the spec as the app would: layers, locks, bounds, plugins, tasks, hooks (R4)."""
|
|
40
|
+
harness = load_harness(path, env, profile_dir or [])
|
|
41
|
+
level = harness.spec.level
|
|
42
|
+
if output == "json":
|
|
43
|
+
body = {
|
|
44
|
+
"ok": True,
|
|
45
|
+
"name": harness.name,
|
|
46
|
+
"level": level.level.value,
|
|
47
|
+
"required": level.required.value,
|
|
48
|
+
"tasks": harness.tasks,
|
|
49
|
+
"layers": [p.describe() for p in harness.spec.layers],
|
|
50
|
+
}
|
|
51
|
+
typer.echo(json.dumps(body, indent=2))
|
|
52
|
+
return
|
|
53
|
+
typer.echo(f"ok: {harness.name} is valid")
|
|
54
|
+
typer.echo(f" adoption level {level.level.value} (required: {level.required.value})")
|
|
55
|
+
typer.echo(f" layers: {', '.join(p.describe() for p in harness.spec.layers)}")
|
|
56
|
+
typer.echo(f" tasks: {', '.join(harness.tasks) or '(none)'}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@spec_app.command("explain")
|
|
60
|
+
def explain(
|
|
61
|
+
path: Path_ = None,
|
|
62
|
+
env: Env = None,
|
|
63
|
+
profile_dir: ProfileDirs = None,
|
|
64
|
+
prefix: Annotated[
|
|
65
|
+
str | None, typer.Option("--path", help="Only paths starting with this.")
|
|
66
|
+
] = None,
|
|
67
|
+
output: Format = "text",
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Show every effective value, the layer it came from, and its locks and bounds."""
|
|
70
|
+
effective = load_effective(str(spec_path(path)), env, profile_dir or [])
|
|
71
|
+
rows = [r for r in effective.explain() if prefix is None or r.path.startswith(prefix)]
|
|
72
|
+
if output == "json":
|
|
73
|
+
body = [
|
|
74
|
+
{
|
|
75
|
+
"path": r.path,
|
|
76
|
+
"value": r.value,
|
|
77
|
+
"layer": r.origin.layer if r.origin else None,
|
|
78
|
+
"locked_by": r.locked_by.declared_by.layer if r.locked_by else None,
|
|
79
|
+
}
|
|
80
|
+
for r in rows
|
|
81
|
+
]
|
|
82
|
+
typer.echo(json.dumps(body, indent=2))
|
|
83
|
+
return
|
|
84
|
+
typer.echo(render_explain(rows))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@spec_app.command("diff")
|
|
88
|
+
def diff(
|
|
89
|
+
before: Annotated[str, typer.Argument(help="Old spec: a path, or git:<ref>:<path>.")],
|
|
90
|
+
after: Annotated[str, typer.Argument(help="New spec: a path, or git:<ref>:<path>.")],
|
|
91
|
+
env: Env = None,
|
|
92
|
+
profile_dir: ProfileDirs = None,
|
|
93
|
+
exit_code: Annotated[
|
|
94
|
+
bool, typer.Option("--exit-code", help="Exit 1 when the effective specs differ.")
|
|
95
|
+
] = False,
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Compare two effective specs as Markdown: values, locks and adoption level."""
|
|
98
|
+
dirs = profile_dir or []
|
|
99
|
+
result = diff_specs(load_effective(before, env, dirs), load_effective(after, env, dirs))
|
|
100
|
+
typer.echo(render_diff(result))
|
|
101
|
+
if exit_code and not result.empty:
|
|
102
|
+
raise typer.Exit(code=1)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def hooks(
|
|
106
|
+
path: Path_ = None,
|
|
107
|
+
env: Env = None,
|
|
108
|
+
profile_dir: ProfileDirs = None,
|
|
109
|
+
task: Annotated[str | None, typer.Option("--task", help="Show a task's chains.")] = None,
|
|
110
|
+
) -> None:
|
|
111
|
+
"""Show the middleware on every hook point, in the order it runs (DESIGNS No. 8)."""
|
|
112
|
+
harness = load_harness(path, env, profile_dir or [])
|
|
113
|
+
reaches = harness.runner_of(task).reaches if task is not None else None
|
|
114
|
+
title = f"{harness.name} hooks" + (f" for task {task}" if task else "")
|
|
115
|
+
table = Table(title=escape(title), box=box.ASCII)
|
|
116
|
+
for column in ("hook", "order", "middleware", "band", "fail mode", "timeout"):
|
|
117
|
+
table.add_column(column)
|
|
118
|
+
for hook, chain in harness.hooks(task).items():
|
|
119
|
+
label = hook.value
|
|
120
|
+
if reaches is not None and hook not in reaches:
|
|
121
|
+
label += " (not reached)"
|
|
122
|
+
if not chain:
|
|
123
|
+
table.add_row(label, "-", "-", "", "", "")
|
|
124
|
+
for position, registration in enumerate(chain, start=1):
|
|
125
|
+
timeout = f"{registration.timeout.total_seconds():g}s" if registration.timeout else ""
|
|
126
|
+
table.add_row(
|
|
127
|
+
label if position == 1 else "",
|
|
128
|
+
str(position),
|
|
129
|
+
escape(registration.name),
|
|
130
|
+
registration.band.value,
|
|
131
|
+
registration.fail_mode,
|
|
132
|
+
timeout,
|
|
133
|
+
)
|
|
134
|
+
console = _console()
|
|
135
|
+
console.print(table)
|
|
136
|
+
direction = {h.value: h.direction.value for h in HookPoint}
|
|
137
|
+
console.print(
|
|
138
|
+
"inbound hooks run baseline -> task; outbound hooks run task -> baseline: "
|
|
139
|
+
+ ", ".join(sorted(k for k, v in direction.items() if v == "outbound"))
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@contracts_app.command("export")
|
|
144
|
+
def export(
|
|
145
|
+
out: Annotated[Path, typer.Argument(help="Directory for the JSON Schemas.")] = Path(
|
|
146
|
+
"contracts/schemas"
|
|
147
|
+
),
|
|
148
|
+
check: Annotated[bool, typer.Option("--check", help="Report drift; write nothing.")] = False,
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Write the JSON Schema of every contract (harness.yaml, events, outcomes, ...)."""
|
|
151
|
+
if check:
|
|
152
|
+
problems = stale_schemas(out)
|
|
153
|
+
for problem in problems:
|
|
154
|
+
typer.echo(problem)
|
|
155
|
+
if problems:
|
|
156
|
+
raise typer.Exit(code=1)
|
|
157
|
+
typer.echo(f"ok: the schemas in {out} are up to date")
|
|
158
|
+
return
|
|
159
|
+
written = write_schemas(out)
|
|
160
|
+
typer.echo(f"wrote {len(written)} schemas to {out}")
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# A local stack for this app, written by `harnessweaver dev` (ADR-0039): LiteLLM Proxy
|
|
2
|
+
# (config from `harnessweaver gateway litellm`), an OpenTelemetry collector that sends
|
|
3
|
+
# traces to Langfuse, and harnessd serving the spec. Secrets and the provider models
|
|
4
|
+
# are in .env, next to this file; nothing here is meant to be committed.
|
|
5
|
+
#
|
|
6
|
+
# docker compose up -d (or: harnessweaver dev --up)
|
|
7
|
+
#
|
|
8
|
+
# harnessd and the collector share LiteLLM's network, so the URLs a spec uses on the
|
|
9
|
+
# developer's machine (http://localhost:4000 for the gateway, :4318 for OTLP) work
|
|
10
|
+
# unchanged inside the containers too. Every port is published on 127.0.0.1 only (S6),
|
|
11
|
+
# and every image is pinned by digest (S5).
|
|
12
|
+
|
|
13
|
+
name: harnessweaver-dev
|
|
14
|
+
|
|
15
|
+
x-langfuse-env: &langfuse-env
|
|
16
|
+
NEXTAUTH_URL: http://localhost:3000
|
|
17
|
+
DATABASE_URL: postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
|
|
18
|
+
SALT: ${SALT}
|
|
19
|
+
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
|
|
20
|
+
TELEMETRY_ENABLED: "false"
|
|
21
|
+
CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse:9000
|
|
22
|
+
CLICKHOUSE_URL: http://clickhouse:8123
|
|
23
|
+
CLICKHOUSE_USER: clickhouse
|
|
24
|
+
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
|
25
|
+
CLICKHOUSE_CLUSTER_ENABLED: "false"
|
|
26
|
+
LANGFUSE_S3_EVENT_UPLOAD_BUCKET: langfuse
|
|
27
|
+
LANGFUSE_S3_EVENT_UPLOAD_REGION: auto
|
|
28
|
+
LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: minio
|
|
29
|
+
LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
|
|
30
|
+
LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: http://minio:9000
|
|
31
|
+
LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: "true"
|
|
32
|
+
LANGFUSE_S3_EVENT_UPLOAD_PREFIX: events/
|
|
33
|
+
LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: langfuse
|
|
34
|
+
LANGFUSE_S3_MEDIA_UPLOAD_REGION: auto
|
|
35
|
+
LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: minio
|
|
36
|
+
LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
|
|
37
|
+
LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: http://localhost:9090
|
|
38
|
+
LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: "true"
|
|
39
|
+
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: media/
|
|
40
|
+
REDIS_HOST: redis
|
|
41
|
+
REDIS_PORT: "6379"
|
|
42
|
+
REDIS_AUTH: ${REDIS_AUTH}
|
|
43
|
+
|
|
44
|
+
x-langfuse-depends: &langfuse-depends
|
|
45
|
+
postgres: { condition: service_healthy }
|
|
46
|
+
clickhouse: { condition: service_healthy }
|
|
47
|
+
minio: { condition: service_healthy }
|
|
48
|
+
redis: { condition: service_healthy }
|
|
49
|
+
|
|
50
|
+
services:
|
|
51
|
+
litellm:
|
|
52
|
+
image: ghcr.io/berriai/litellm:v1.104.0@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839
|
|
53
|
+
command: ["--config", "/etc/litellm/config.yaml", "--port", "4000"]
|
|
54
|
+
env_file: .env
|
|
55
|
+
volumes:
|
|
56
|
+
- ./litellm.yaml:/etc/litellm/config.yaml:ro
|
|
57
|
+
ports:
|
|
58
|
+
- 127.0.0.1:4000:4000 # LiteLLM Proxy
|
|
59
|
+
- 127.0.0.1:4317:4317 # OTLP gRPC (the collector)
|
|
60
|
+
- 127.0.0.1:4318:4318 # OTLP HTTP (the collector)
|
|
61
|
+
- 127.0.0.1:7070:7070 # harnessd
|
|
62
|
+
restart: unless-stopped
|
|
63
|
+
|
|
64
|
+
otel-collector:
|
|
65
|
+
image: otel/opentelemetry-collector-contrib:0.162.0@sha256:39923a8e431bd1f57be82411999d389fcfe40857492e4365456d97a4c1f74be6
|
|
66
|
+
command: ["--config", "/etc/otelcol/config.yaml"]
|
|
67
|
+
network_mode: service:litellm
|
|
68
|
+
environment:
|
|
69
|
+
LANGFUSE_OTLP_AUTH: ${LANGFUSE_OTLP_AUTH}
|
|
70
|
+
volumes:
|
|
71
|
+
- ./otel-collector.yaml:/etc/otelcol/config.yaml:ro
|
|
72
|
+
depends_on: [litellm, langfuse-web]
|
|
73
|
+
restart: unless-stopped
|
|
74
|
+
|
|
75
|
+
harnessd:
|
|
76
|
+
# Build it from the HarnessWeaver repository: docker build -f deploy/harnessd/Dockerfile -t harnessd:dev .
|
|
77
|
+
image: ${HARNESSD_IMAGE:-harnessd:dev}
|
|
78
|
+
network_mode: service:litellm
|
|
79
|
+
env_file: .env
|
|
80
|
+
environment:
|
|
81
|
+
HARNESSWEAVER_ENV: "__env__"
|
|
82
|
+
HARNESSWEAVER_PROFILE_PATH: "__profile_path__" # the folders --profile-dir named
|
|
83
|
+
volumes:
|
|
84
|
+
- ../..:/app:ro # the app: its spec, overlays, profiles and code
|
|
85
|
+
__profile_volumes__ command: ["__spec__"]
|
|
86
|
+
depends_on: [litellm]
|
|
87
|
+
restart: unless-stopped
|
|
88
|
+
|
|
89
|
+
langfuse-web:
|
|
90
|
+
image: docker.io/langfuse/langfuse:4@sha256:ce6c636df92db6638f15beb7d826f0460a51d04fcfe2e55132fcc8aad9e1bef8
|
|
91
|
+
depends_on: *langfuse-depends
|
|
92
|
+
ports:
|
|
93
|
+
- 127.0.0.1:3000:3000
|
|
94
|
+
environment:
|
|
95
|
+
<<: *langfuse-env
|
|
96
|
+
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET}
|
|
97
|
+
LANGFUSE_S3_MEDIA_UPLOAD_INTERNAL_ENDPOINT: http://minio:9000
|
|
98
|
+
LANGFUSE_INIT_ORG_ID: harnessweaver
|
|
99
|
+
LANGFUSE_INIT_ORG_NAME: HarnessWeaver
|
|
100
|
+
LANGFUSE_INIT_PROJECT_ID: dev
|
|
101
|
+
LANGFUSE_INIT_PROJECT_NAME: dev
|
|
102
|
+
LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY}
|
|
103
|
+
LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_SECRET_KEY}
|
|
104
|
+
LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_USER_EMAIL}
|
|
105
|
+
LANGFUSE_INIT_USER_NAME: dev
|
|
106
|
+
LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_USER_PASSWORD}
|
|
107
|
+
restart: unless-stopped
|
|
108
|
+
|
|
109
|
+
langfuse-worker:
|
|
110
|
+
image: docker.io/langfuse/langfuse-worker:4@sha256:68549f1167370de7e601114e592dbc86aac13216d90c5193063a2d9d83d9ec57
|
|
111
|
+
depends_on: *langfuse-depends
|
|
112
|
+
environment: *langfuse-env
|
|
113
|
+
restart: unless-stopped
|
|
114
|
+
|
|
115
|
+
postgres:
|
|
116
|
+
image: docker.io/postgres:17@sha256:2d2b8998d31037bf721cfdf764d76ba74171b4fab3431b7f72c27c56ddbdf9e3
|
|
117
|
+
environment:
|
|
118
|
+
POSTGRES_USER: postgres
|
|
119
|
+
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
120
|
+
POSTGRES_DB: postgres
|
|
121
|
+
volumes:
|
|
122
|
+
- postgres_data:/var/lib/postgresql/data
|
|
123
|
+
healthcheck:
|
|
124
|
+
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
|
125
|
+
interval: 3s
|
|
126
|
+
timeout: 3s
|
|
127
|
+
retries: 10
|
|
128
|
+
restart: unless-stopped
|
|
129
|
+
|
|
130
|
+
clickhouse:
|
|
131
|
+
image: docker.io/clickhouse/clickhouse-server:25.12@sha256:8a790dd3468db22b1d4e7b18a176f378ff5ff6053b9c48dd4ea1fa71a24c5ba6
|
|
132
|
+
user: "101:101"
|
|
133
|
+
environment:
|
|
134
|
+
CLICKHOUSE_DB: default
|
|
135
|
+
CLICKHOUSE_USER: clickhouse
|
|
136
|
+
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
|
137
|
+
volumes:
|
|
138
|
+
- clickhouse_data:/var/lib/clickhouse
|
|
139
|
+
- clickhouse_logs:/var/log/clickhouse-server
|
|
140
|
+
healthcheck:
|
|
141
|
+
test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
|
|
142
|
+
interval: 5s
|
|
143
|
+
timeout: 5s
|
|
144
|
+
retries: 10
|
|
145
|
+
restart: unless-stopped
|
|
146
|
+
|
|
147
|
+
minio:
|
|
148
|
+
image: cgr.dev/chainguard/minio:latest@sha256:cff197fef764642223ef9f4444c903d51aee9b544a7be8bc007fef6b7b00d136
|
|
149
|
+
entrypoint: sh
|
|
150
|
+
# Langfuse's bucket must exist before it starts.
|
|
151
|
+
command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
|
|
152
|
+
environment:
|
|
153
|
+
MINIO_ROOT_USER: minio
|
|
154
|
+
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
|
155
|
+
ports:
|
|
156
|
+
- 127.0.0.1:9090:9000 # media uploads from the browser
|
|
157
|
+
volumes:
|
|
158
|
+
- minio_data:/data
|
|
159
|
+
healthcheck:
|
|
160
|
+
test: ["CMD", "mc", "ready", "local"]
|
|
161
|
+
interval: 1s
|
|
162
|
+
timeout: 5s
|
|
163
|
+
retries: 5
|
|
164
|
+
restart: unless-stopped
|
|
165
|
+
|
|
166
|
+
redis:
|
|
167
|
+
image: docker.io/redis:7@sha256:4fa24486b8bcca8eec45ee0eb166edc674795e53a2b53d1a9ef263eecebaac85
|
|
168
|
+
command: ["--requirepass", "${REDIS_AUTH}", "--maxmemory-policy", "noeviction"]
|
|
169
|
+
volumes:
|
|
170
|
+
- redis_data:/data
|
|
171
|
+
healthcheck:
|
|
172
|
+
test: ["CMD", "redis-cli", "-a", "${REDIS_AUTH}", "ping"]
|
|
173
|
+
interval: 3s
|
|
174
|
+
timeout: 10s
|
|
175
|
+
retries: 10
|
|
176
|
+
restart: unless-stopped
|
|
177
|
+
|
|
178
|
+
volumes:
|
|
179
|
+
postgres_data:
|
|
180
|
+
clickhouse_data:
|
|
181
|
+
clickhouse_logs:
|
|
182
|
+
minio_data:
|
|
183
|
+
redis_data:
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# The dev stack's OpenTelemetry collector, written by `harnessweaver dev`: OTLP in from
|
|
2
|
+
# the app, harnessd and LiteLLM, traces out to the local Langfuse (OTLP over HTTP;
|
|
3
|
+
# Langfuse doesn't take gRPC). LANGFUSE_OTLP_AUTH comes from .env.
|
|
4
|
+
receivers:
|
|
5
|
+
otlp:
|
|
6
|
+
protocols:
|
|
7
|
+
grpc: { endpoint: 0.0.0.0:4317 }
|
|
8
|
+
http: { endpoint: 0.0.0.0:4318 }
|
|
9
|
+
|
|
10
|
+
processors:
|
|
11
|
+
batch: {}
|
|
12
|
+
|
|
13
|
+
exporters:
|
|
14
|
+
otlp_http/langfuse:
|
|
15
|
+
endpoint: http://langfuse-web:3000/api/public/otel
|
|
16
|
+
headers:
|
|
17
|
+
Authorization: "Basic ${env:LANGFUSE_OTLP_AUTH}"
|
|
18
|
+
x-langfuse-ingestion-version: "4"
|
|
19
|
+
|
|
20
|
+
service:
|
|
21
|
+
pipelines:
|
|
22
|
+
traces:
|
|
23
|
+
receivers: [otlp]
|
|
24
|
+
processors: [batch]
|
|
25
|
+
exporters: [otlp_http/langfuse]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# __dist__
|
|
2
|
+
|
|
3
|
+
A HarnessWeaver plugin. It contributes everything under the namespace `__ns__`; the sample contribution is the guard `__ns__.banned_words`.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
uv sync # install it, with pytest
|
|
7
|
+
uv run pytest # its tests and the HarnessWeaver conformance suite
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Use it from any app's `harness.yaml` once it is installed next to `harnessweaver`:
|
|
11
|
+
|
|
12
|
+
```yaml
|
|
13
|
+
guardrails:
|
|
14
|
+
input:
|
|
15
|
+
- { use: __ns__.banned_words, words: [lottery, wire transfer], mode: log_only }
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
New guards start in shadow mode (`mode: log_only`) and are promoted to `enforce` once evals back them. If the org baseline locks `plugins.allow`, ask the platform team to add `__ns__` to it.
|
|
19
|
+
|
|
20
|
+
## Add a contribution
|
|
21
|
+
|
|
22
|
+
1. Write it in `src/__ns___harness/` with its decorator: `@hw.provider(port=..., name="__ns__.<name>", ...)`, `@hw.middleware(...)`, `@hw.decider("<name>")`, `@hw.runner(...)` or `@hw.adapter(...)`. A contribution whose config has required fields declares `sample=` settings for the conformance suite.
|
|
23
|
+
2. List it in `plugin = hw.Plugin(...)` in `src/__ns___harness/__init__.py`.
|
|
24
|
+
3. Test it, and run `uv run pytest`: the conformance suite checks every contribution.
|
|
25
|
+
|
|
26
|
+
Build only against the public surface: `harnessweaver` (`hw.Plugin`, the decorators and the contract models), `harnessweaver.ports` and `harnessweaver.testing`. It is versioned with the contract (`contract=">=1.0,<2"`), so the plugin keeps working across HarnessWeaver releases within `harnessweaver/v1`.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.32"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "__dist__"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A HarnessWeaver plugin: guards and other contributions under the __ns__ namespace."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
dependencies = ["harnessweaver>=0.1.0"]
|
|
12
|
+
|
|
13
|
+
# The one registration a plugin needs: its namespace, and where its hw.Plugin is (DESIGNS No. 13).
|
|
14
|
+
[project.entry-points."harnessweaver.plugins"]
|
|
15
|
+
__ns__ = "__ns___harness:plugin"
|
|
16
|
+
|
|
17
|
+
[dependency-groups]
|
|
18
|
+
dev = ["pytest>=9"]
|
|
19
|
+
|
|
20
|
+
[tool.hatch.build.targets.wheel]
|
|
21
|
+
packages = ["src/__ns___harness"]
|
|
22
|
+
|
|
23
|
+
[tool.pytest.ini_options]
|
|
24
|
+
testpaths = ["tests"]
|
|
25
|
+
# Every run also runs the HarnessWeaver conformance suite on this plugin (RULES T5).
|
|
26
|
+
addopts = ["--import-mode=importlib", "--harnessweaver-conformance=__ns__"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""__dist__: a HarnessWeaver plugin under the namespace ``__ns__``.
|
|
2
|
+
|
|
3
|
+
Installing it registers nothing and imports nothing: HarnessWeaver finds it
|
|
4
|
+
through the ``harnessweaver.plugins`` entry point and imports it only when a
|
|
5
|
+
spec names one of its contributions, such as ``use: __ns__.banned_words``
|
|
6
|
+
(DESIGNS No. 13, RULES S8). So this module must not do I/O when it is imported.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import harnessweaver as hw
|
|
10
|
+
from __ns___harness.guards import BannedWords, BannedWordsConfig
|
|
11
|
+
|
|
12
|
+
plugin = hw.Plugin(
|
|
13
|
+
namespace="__ns__",
|
|
14
|
+
version="0.1.0",
|
|
15
|
+
contract=">=1.0,<2",
|
|
16
|
+
providers=[BannedWords],
|
|
17
|
+
description="Guards for __ns__ apps.",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__all__ = ["BannedWords", "BannedWordsConfig", "plugin"]
|