open-interpretability 0.0.20__py3-none-any.whl

openinterp/cli_costs.py

@@ -0,0 +1,257 @@
+"""``open-interpretability costs`` and ``open-interpretability usage``: what experiments cost.
+
+``open-interpretability costs`` summarises costs from ledgers of commands or from the registry's
+records. ``open-interpretability usage`` records what an external answerer used.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import click
+
+from openinterp.cli import cli
+from openinterp.core.paths import experiments_dir
+
+
+def read_ledger(paths) -> list[dict]:
+    """Every entry of the ledgers at ``paths``; a missing ledger counts as empty."""
+    entries = []
+    for path in paths:
+        if not path.exists():
+            continue  # an iteration that ran no openinterp command
+        for line in path.read_text().splitlines():
+            if line.strip():
+                entries.append(json.loads(line))
+    return entries
+
+
+def read_session(path: Path) -> dict | None:
+    """The result object of ``claude -p --output-format json``, or ``None``."""
+    try:
+        data = json.loads(path.read_text())
+    except (OSError, ValueError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def reported_usd(entries: list[dict], sessions: list[dict]) -> tuple[float, int]:
+    """Dollars reported by the LLM calls and driving sessions, and the count left unpriced."""
+    calls = [c for e in entries for c in e.get("llm_calls", [])]
+    priced = [c["cost_usd"] for c in calls if c.get("cost_usd") is not None]
+    priced += [s["total_cost_usd"] for s in sessions if s.get("total_cost_usd") is not None]
+    unpriced = sum(c.get("cost_usd") is None for c in calls)
+    unpriced += sum(s.get("total_cost_usd") is None for s in sessions)
+    return sum(priced), unpriced
+
+
+def summarise(entries: list[dict], sessions: list[dict]) -> str:
+    """Tables of wall time per command and stage, and of LLM usage, from ledgers and sessions."""
+    commands: dict[str, list[float]] = defaultdict(list)
+    stages: dict[str, list[float]] = defaultdict(list)
+    for e in entries:
+        commands[e["command"]].append(e["seconds"])
+        for s in e.get("stages", []):
+            stages[s["stage"]].append(s["seconds"])
+    lines = _table("Commands", commands) + _table("Stages", stages)
+    calls: dict[tuple[str, str], list[dict]] = defaultdict(list)
+    for e in entries:
+        for c in e.get("llm_calls", []):
+            calls[(c["agent"], c["model"])].append(c)
+    if calls:
+        lines.append("LLM calls:")
+        for (agent, model), cs in sorted(calls.items()):
+            lines.append(f"  {agent:<9} {model:<34} ×{len(cs):<3} {_usage(cs)}")
+    if sessions:
+        lines.append(f"Driver: {_session(sessions)}")
+    total = sum(e["seconds"] for e in entries)
+    known = [c["cost_usd"] for cs in calls.values() for c in cs if c.get("cost_usd") is not None]
+    known += [s["total_cost_usd"] for s in sessions if s.get("total_cost_usd") is not None]
+    cost = f", ${sum(known):.2f} reported" if known else ""
+    took = f"{total / 60:.1f} min" if total >= 60 else f"{total:.1f} s"
+    lines.append(f"Total: {took} in openinterp commands{cost}")
+    return "\n".join(lines)
+
+
+def _table(title: str, groups: dict[str, list[float]]) -> list[str]:
+    rows = sorted(groups.items(), key=lambda kv: -sum(kv[1]))
+    head = [f"{title} (wall time, s):"] if rows else []
+    return head + [f"  {name:<22} ×{len(secs):<3} {sum(secs):9.1f}" for name, secs in rows]
+
+
+def _usage(calls: list[dict]) -> str:
+    def total(key):
+        values = [c[key] for c in calls if c.get(key) is not None]
+        return sum(values) if values else None
+
+    parts = []
+    if (chars := total("prompt_chars")) is not None:
+        parts.append(f"{chars / 1000:.0f}k prompt chars")
+    prompt, completion = total("prompt_tokens"), total("completion_tokens")
+    if prompt is not None or completion is not None:
+        cached = total("cached_tokens")
+        hit = f" ({cached:,} cached)" if cached else ""
+        parts.append(f"{prompt or 0:,}{hit} + {completion or 0:,} tokens")
+    if (cost := total("cost_usd")) is not None:
+        parts.append(f"${cost:.3f}")
+    if any(c.get("external") and c.get("prompt_tokens") is None for c in calls):
+        parts.append("external without a usage file: no token counts")
+    return ", ".join(parts)
+
+
+def _session(sessions: list[dict]) -> str:
+    usage: dict[str, int] = defaultdict(int)
+    for s in sessions:
+        for key, value in (s.get("usage") or {}).items():
+            if isinstance(value, int):
+                usage[key] += value
+    minutes = sum(s.get("duration_ms") or 0 for s in sessions) / 60000
+    turns = sum(s.get("num_turns") or 0 for s in sessions)
+    read, written = usage["cache_read_input_tokens"], usage["cache_creation_input_tokens"]
+    tokens = (
+        f"{usage['input_tokens'] + read + written:,} input ({read:,} cache reads) + "
+        f"{usage['output_tokens']:,} output tokens"
+    )
+    costs = [s["total_cost_usd"] for s in sessions if s.get("total_cost_usd") is not None]
+    cost = f", ${sum(costs):.2f}" if costs else ""
+    n = len(sessions)
+    return f"{n} session{'s' * (n != 1)}, {minutes:.1f} min, {turns} turns, {tokens}{cost}"
+
+
+def registry_costs(wiki_dir: Path) -> str:
+    """Cost per experiment by role and model, over the records in ``wiki_dir``."""
+    from openinterp.builder.compiler import load_records
+    from openinterp.core.costs import by_agent
+
+    records = load_records(experiments_dir(wiki_dir))
+    rows = by_agent(
+        (
+            r.verdict.status if r.verdict else None,
+            r.submission.provenance and r.submission.provenance.costs,
+        )
+        for r in records
+    )
+    if not rows:
+        return f"No LLM calls recorded in {len(records)} records."
+
+    def usd(v):
+        return f"${v:.3f}" if v is not None else "–"
+
+    lines = [
+        f"LLM cost per experiment ({len(records)} records; priced = every call has a cost):",
+        f"  {'role':<9} {'model':<40} {'priced':>11} {'mean':>8} {'per VERIFIED':>13}",
+    ]
+    for r in rows:
+        priced = f"{r['priced']}/{r['experiments']}"
+        lines.append(
+            f"  {r['agent']:<9} {r['model']:<40} {priced:>11} {usd(r['mean_usd']):>8} "
+            f"{usd(r['usd_per_verified']):>13}"
+        )
+    return "\n".join(lines)
+
+
+@cli.command()
+@click.argument("ledgers", nargs=-1, type=click.Path(dir_okay=False, path_type=Path))
+@click.option(
+    "--session",
+    "sessions",
+    multiple=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="`claude -p --output-format json` output of the driving session (repeatable).",
+)
+@click.option(
+    "--wiki-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Summarise the registry's records instead: LLM cost per experiment by role and "
+    "model, mean and per VERIFIED result.",
+)
+@click.option(
+    "--usd",
+    is_flag=True,
+    help="Print only the dollars reported, as one number. The count of unpriced calls "
+    "goes to stderr. scripts/experiment-loop.sh compares it with its budgets.",
+)
+def costs(ledgers, sessions, wiki_dir, usd):
+    """Summarise wall time and LLM usage, from ledgers or from the registry.
+
+    The ledgers are $OPENINTERP_COSTS_LOG files. With --wiki-dir, it gives the registry's
+    LLM costs by role and model.
+
+    Set OPENINTERP_COSTS_LOG=<file> and every openinterp command appends what it measured
+    there: its wall time, its stages and its LLM calls. Records hold the same numbers for
+    their own experiment, in `submission.provenance.costs`. Reported, never gated.
+    """
+    if wiki_dir is not None:
+        if ledgers or sessions:
+            raise click.UsageError("--wiki-dir summarises records; drop the ledgers.")
+        click.echo(registry_costs(wiki_dir))
+        return
+    if not ledgers and not sessions:
+        if usd:  # no experiment has run yet, so nothing is spent
+            click.echo("0.0000")
+            return
+        raise click.UsageError("Pass at least one ledger, --session file or --wiki-dir.")
+    found = [s for s in (read_session(p) for p in sessions) if s is not None]
+    if usd:
+        dollars, unpriced = reported_usd(read_ledger(ledgers), found)
+        if unpriced:
+            click.echo(f"{unpriced} unpriced calls or sessions are not counted", err=True)
+        click.echo(f"{dollars:.4f}")
+        return
+    click.echo(summarise(read_ledger(ledgers), found))
+
+
+@cli.command()
+@click.argument("agent")
+@click.option(
+    "--answer",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="The answer the subagent wrote, e.g. exchange/ev.hypothesis.json.",
+)
+def usage(agent, answer):
+    """Record a Claude Code subagent's token usage next to the answer it wrote.
+
+    AGENT is the agent ID that Claude Code reports when the subagent finishes. It can also
+    be the path of the subagent's transcript:
+    ~/.claude/projects/<project>/<session>/subagents/agent-<id>.jsonl.
+    Writes <answer>.usage.json. `audit --hypothesis`, `review --probes` and
+    `score --task` read it to record the call's tokens and cost. They price it at list
+    price for the model that they record. That is after $OPENINTERP_MODEL_ALIASES, so a
+    proxied model is priced as the model that actually answered.
+    """
+    from openinterp.core.costs import usage_path
+    from openinterp.core.paths import PROBES_SUFFIX
+    from openinterp.core.transcripts import (
+        find_transcript,
+        forbidden_reads,
+        usage_from_transcript,
+    )
+
+    try:
+        transcript = find_transcript(agent)
+    except FileNotFoundError as exc:
+        raise click.BadParameter(str(exc), param_hint="AGENT") from exc
+    found = usage_from_transcript(transcript)
+    if answer.name.endswith(PROBES_SUFFIX):  # a reviewer's answer: check its isolation
+        read, tried = forbidden_reads(transcript)
+        found = found.model_copy(update={"forbidden_reads": read})
+        if tried:
+            click.echo(f"The reviewer tried to read blocked paths: {', '.join(tried)}", err=True)
+        if found.forbidden_reads:
+            click.echo(
+                "warning: the reviewer read blocked paths, so `review --probes` will refuse "
+                f"{answer}: {', '.join(found.forbidden_reads)}. Run a new reviewer.",
+                err=True,
+            )
+    out = usage_path(answer)
+    out.write_text(found.model_dump_json(indent=2) + "\n")
+    tokens = found.input_tokens + found.cache_read_tokens
+    tokens += found.cache_write_5m_tokens + found.cache_write_1h_tokens
+    click.echo(
+        f"{found.model}: {found.requests} requests, {tokens:,} input "
+        f"({found.cache_read_tokens:,} cache reads) + {found.output_tokens:,} output tokens. "
+        f"Wrote {out}"
+    )

openinterp/cli_coverage.py

@@ -0,0 +1,76 @@
+"""The ``coverage`` command: how much of each layer's effect the VERIFIED poles keep."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+import click
+
+from openinterp.cli import cli, wiki_dir_option, with_model_options
+
+
+@cli.command()
+@click.option("--model", "model_name", required=True, help="Hugging Face model ID.")
+@click.option(
+    "--corpus",
+    type=click.Path(exists=True, path_type=Path),
+    default="corpus.jsonl",
+    show_default=True,
+    help="Text for the windows.",
+)
+@click.option(
+    "--windows",
+    default=64,
+    show_default=True,
+    type=click.IntRange(1),
+    help="Windows to fit the means on, and as many again to measure the loss on.",
+)
+@click.option("--window-tokens", default=128, show_default=True, type=click.IntRange(8))
+@click.option("--batch", default=4, show_default=True, type=click.IntRange(1))
+@click.option("--seed", default=0, show_default=True, type=int)
+@click.option("--layers", help="Comma-separated layers (default: every layer).")
+@click.option(
+    "--out",
+    type=click.Path(dir_okay=False, path_type=Path),
+    default="docs/analyses/registry-coverage.jsonl",
+    show_default=True,
+    help="Rows are appended here, one per layer.",
+)
+@wiki_dir_option()
+@with_model_options
+def coverage(model_name, corpus, windows, window_tokens, batch, seed, layers, out, wiki_dir,
+             device, dtype):  # fmt: skip
+    """Measure the share of each MLP layer's loss effect that the VERIFIED poles keep.
+
+    Reported, not gated. See openinterp/coverage.py for the method.
+    """
+    from openinterp import __version__
+    from openinterp.cli import _load_runner
+    from openinterp.cli_evidence import read_corpus
+    from openinterp.coverage import layer_row, mean_loss, verified_poles
+    from openinterp.coverage import windows as draw_windows
+
+    verified = verified_poles(model_name, wiki_dir)
+    runner = _load_runner(model_name, device, dtype)
+    texts, corpus_name, corpus_sha, _ = read_corpus(corpus)
+    fit, test = draw_windows(runner, texts, windows, window_tokens, seed)
+    chosen = [int(x) for x in layers.split(",")] if layers else range(runner.n_layers)
+    clean = mean_loss(runner, test, batch)
+    click.echo(f"Clean loss {clean:.4f} on {test.numel()} tokens", err=True)
+    meta = {"model": model_name, "corpus": corpus_name, "corpus_sha256": corpus_sha,
+            "windows": windows, "window_tokens": window_tokens, "seed": seed,
+            "dtype": dtype, "openinterp": __version__,
+            "date": datetime.now(UTC).strftime("%Y-%m-%d")}  # fmt: skip
+    out.parent.mkdir(parents=True, exist_ok=True)
+    for i, layer in enumerate(chosen, 1):
+        row = meta | layer_row(
+            runner, fit, test, layer, verified.get(layer, set()), clean, batch, seed
+        )
+        with out.open("a") as fh:
+            fh.write(json.dumps(row) + "\n")
+        share = row.get("recovered")
+        click.echo(f"[{i}/{len(chosen)}] L{layer}: {row['verified_poles']} VERIFIED poles, "
+                   f"recovered {'—' if share is None else f'{share:.3f}'}"
+                   f" (random {row.get('recovered_random') or 0:.3f})", err=True)  # fmt: skip

openinterp/cli_crossref.py

@@ -0,0 +1,86 @@
+"""``open-interpretability crossref`` and ``open-interpretability aligned-latents``.
+
+They compare neuron poles with published dictionaries. See :mod:`openinterp.crossref`.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from openinterp.cli import _load_runner, cli, wiki_dir_option
+from openinterp.crossref import DEFAULT_PATH
+
+out_option = click.option(
+    "--out",
+    "path",
+    default=DEFAULT_PATH,
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="JSONL of alignment rows, one per pole and dictionary.",
+)
+
+
+@cli.command("crossref")
+@wiki_dir_option()
+@out_option
+@click.option("-k", "k", default=5, show_default=True, help="Latents kept per side.")
+@click.option("--device", default="auto", show_default=True, help="cpu, cuda, mps or auto.")
+def crossref_command(wiki_dir, path, k, device):
+    """Find the dictionary latents aligned with every archived neuron pole's weights.
+
+    Covers each pole whose model has a dictionary preset at its layer; rows already in
+    --out are skipped, so a run can resume. Weights only: no text is run.
+    """
+    from openinterp.builder.wiki import load_archive
+    from openinterp.crossref import crossref, sort_rows
+
+    def load(model_name, revision):
+        return _load_runner(model_name, device, "float32", revision)
+
+    wrote = crossref(load_archive(wiki_dir), load, path, k=k, echo=click.echo)
+    if path.exists():
+        sort_rows(path)
+    click.echo(f"Aligned {wrote} pole–dictionary pairs → {path}")
+
+
+@cli.command("aligned-latents")
+@click.argument("slug")
+@out_option
+@wiki_dir_option()
+def aligned_latents(slug, path, wiki_dir):
+    """Show a pole's aligned latents with their descriptions.
+
+    The pole is a slug such as Qwen--Qwen2.5-0.5B/L3/N12-positive. The descriptions come
+    from this registry or Neuronpedia.
+    """
+    from openinterp.builder.wiki import load_archive
+    from openinterp.crossref import describe, load_rows, neuronpedia_url, registry_explanations
+
+    rows = load_rows(path).get(slug)
+    if not rows:
+        raise click.ClickException(f"no alignment rows for {slug} in {path}; run crossref")
+    registry = registry_explanations(load_archive(wiki_dir))
+    described = {
+        (id(d["row"]), d["side"], d["match"].feature_idx): d for d in describe(rows, registry)
+    }
+    for row in rows:
+        for side_name in ("writes", "reads"):
+            side = getattr(row, side_name)
+            if side is None:
+                continue
+            click.echo(
+                f"{row.dictionary} {side_name} (chance {side.chance:.2f}, "
+                f"best beats {side.percentile:.0%} of the layer's neurons)"
+            )
+            for m in side.matches:
+                d = described.get((id(row), side_name, m.feature_idx))
+                channel = f" {m.channel}" if m.channel and side_name == "reads" else ""
+                text = (d and d["description"]) or ("below chance" if d is None else "—")
+                if d and d["source"]:
+                    text += f"  [{d['source']}]"
+                url = neuronpedia_url(row, m.feature_idx)
+                click.echo(f"  F{m.feature_idx:<6} {m.cosine:+.3f}{channel}  {text}")
+                if url:
+                    click.echo(f"          {url}")

openinterp/cli_errata.py

@@ -0,0 +1,142 @@
+"""``open-interpretability erratum draft``: write an erratum's line from a rule (issue #3271).
+
+A maintainer gives the bug, the fix commit and the records it covers: a rule, listed
+records, or both. The command lists every record that the erratum covers, so that the
+maintainer checks a list and not a hand-written JSON line. It checks the draft as
+``open-interpretability check`` will check the file (:mod:`openinterp.errata_checks`).
+Then it prints the line. ``--write`` appends the line to ``wiki/errata.jsonl``. Signing
+stays a separate step, ``open-interpretability sign wiki/errata.jsonl``, after the
+maintainer reads the list.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from datetime import date as Date
+from pathlib import Path
+
+import click
+
+from openinterp.cli import cli, wiki_dir_option
+
+KINDS = ("mlp_neuron_pole", "attn_head", "dictionary_feature")
+
+
+@cli.group()
+def erratum() -> None:
+    """Draft errata for signed records (docs/PROTOCOL.md §5)."""
+
+
+def _edit(text: str) -> dict:
+    """``PATH=JSON`` as an edit; a value that isn't JSON is a string."""
+    path, sep, value = text.partition("=")
+    if not sep:
+        raise click.BadParameter(f"{text!r} is not PATH=VALUE", param_hint="--edit")
+    try:
+        return {"path": path, "value": json.loads(value)}
+    except json.JSONDecodeError:
+        return {"path": path, "value": value}
+
+
+def _rule(**fields) -> dict | None:
+    """The rule's fields that were given, or ``None`` when none was."""
+    rule = {k: (list(v) if isinstance(v, tuple) else v) for k, v in fields.items()}
+    rule = {k: v for k, v in rule.items() if v not in (None, [])}
+    return rule or None
+
+
+def resolve_commit(ref: str) -> str | None:
+    """The full hash of the commit that ``ref`` names here, or ``None`` if there is none."""
+    done = subprocess.run(
+        ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"],
+        capture_output=True, text=True, check=False,
+    )  # fmt: skip
+    return done.stdout.strip() if done.returncode == 0 else None
+
+
+def draft_erratum(
+    wiki_dir: Path, fields: dict, resolve=resolve_commit
+) -> tuple[object, list[tuple[str, object]]]:
+    """The erratum that ``fields`` describe, and the records it covers as ``(ref, record)``.
+
+    Raises ``click.ClickException`` for each way ``open-interpretability check`` would
+    refuse it. These are an id already used, a fix commit that doesn't exist, and a listed
+    record that isn't in the archive. Also no record covered, or an edit that doesn't apply.
+    ``fields["fixed_in"]`` can be any ref, such as a branch or ``HEAD``. The erratum holds
+    the hash of its commit.
+    """
+    from pydantic import ValidationError
+
+    from openinterp.builder.wiki import load_archive_paths
+    from openinterp.core.errata import Erratum, apply_errata, load_errata, record_ref
+
+    commit = resolve(fields["fixed_in"])
+    if commit is None:
+        raise click.ClickException(f"--fixed-in {fields['fixed_in']} is not a commit here")
+    try:
+        draft = Erratum.model_validate_json(json.dumps(fields | {"fixed_in": commit}))
+    except ValidationError as exc:
+        raise click.ClickException(f"the erratum is not valid: {exc}") from exc
+    if draft.id in {e.id for e in load_errata(wiki_dir)}:
+        raise click.ClickException(f"erratum id {draft.id!r} is already in the errata file")
+    archive = {record_ref(r): r for _, r in load_archive_paths(wiki_dir)}
+    missing = [ref for ref in draft.records if ref not in archive]
+    if missing:
+        raise click.ClickException(f"listed records not in the archive: {', '.join(missing)}")
+    covered = [(ref, r) for ref, r in sorted(archive.items()) if draft.covers(r)]
+    if not covered:
+        raise click.ClickException("the erratum covers no record")
+    for ref, record in covered:
+        try:
+            apply_errata(record, [draft])
+        except ValueError as exc:
+            raise click.ClickException(f"{ref}: {exc}") from exc
+    return draft, covered
+
+
+@erratum.command("draft")
+@wiki_dir_option()
+@click.option("--id", "erratum_id", required=True, help="Unique id: lower case and hyphens.")
+@click.option("--title", required=True, help="One line for the banner.")
+@click.option("--bug", required=True, help="What was wrong, and what it did to records.")
+@click.option("--fixed-in", required=True, help="The commit that fixed the bug.")
+@click.option("--date", "day", default=None, help="The erratum's date (default today).")
+@click.option("--record", "records", multiple=True, help="A record path in wiki/experiments/.")
+@click.option("--model", "models", multiple=True, help="Rule: a model name.")
+@click.option("--kind", "kinds", multiple=True, type=click.Choice(KINDS), help="Rule: unit kind.")
+@click.option("--protocol-version", "protocol_versions", multiple=True, help="Rule: version.")
+@click.option("--harness-version", "harness_versions", multiple=True, help="Rule: version.")
+@click.option("--reviewed-from", help="Rule: first verdict time that matches (ISO, with zone).")
+@click.option("--reviewed-before", help="Rule: verdict time from which records don't match.")
+@click.option("--edit", "edits", multiple=True, help="PATH=JSON: a corrected field.")
+@click.option("--set-aside", is_flag=True, help="The records count toward no status.")
+@click.option("--write", is_flag=True, help="Append the line to wiki/errata.jsonl.")
+def draft(wiki_dir, erratum_id, title, bug, fixed_in, day, records, edits, set_aside, write,
+          **rule):  # fmt: skip
+    """List the records that an erratum covers, check it, and print its line.
+
+    Give a rule (--model, --kind, --protocol-version, --harness-version, --reviewed-from,
+    --reviewed-before), --record paths, or both. Give --edit corrections or --set-aside.
+    Read the list, then sign the file with `open-interpretability sign wiki/errata.jsonl`.
+    """
+    from openinterp.core.errata import errata_path
+
+    fields = {
+        "id": erratum_id, "date": day or Date.today().isoformat(), "title": title, "bug": bug,
+        "fixed_in": fixed_in, "edits": [_edit(e) for e in edits], "set_aside": set_aside,
+        "records": list(records), "rule": _rule(**rule),
+    }  # fmt: skip
+    found, covered = draft_erratum(wiki_dir, fields)
+    for ref, record in covered:
+        verdict = record.verdict
+        status, when = (verdict.status, verdict.timestamp.isoformat()) if verdict else ("-", "-")
+        click.echo(f"  {ref}  {status}  {when}")
+    click.echo(f"The erratum covers {len(covered)} records.")
+    line = found.model_dump_json(exclude_defaults=True)
+    click.echo(line)
+    if write:
+        path = errata_path(wiki_dir)
+        with path.open("a", encoding="utf-8") as f:
+            f.write(line + "\n")
+        click.echo(f"Appended to {path}. Read the list, then: open-interpretability sign {path}")