open-interpretability 0.0.20__py3-none-any.whl

openinterp/cli_build.py

@@ -0,0 +1,128 @@
+"""``open-interpretability build``: compile the archived experiments into the static site."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from openinterp.cli import cli, wiki_dir_option
+from openinterp.core.schemas import COMMITTED_SOURCES
+from openinterp.sisters import DEFAULT_PATH as SISTERS_PATH
+
+
+@cli.command()
+@wiki_dir_option("Experiment archive (wiki/experiments/**/<id>.json).", exists=True)
+@click.option(
+    "--output-dir", default=Path("dist"), show_default=True, type=click.Path(path_type=Path)
+)
+@click.option(
+    "--wiring",
+    "wiring_path",
+    default=Path("docs/analyses/wiring.jsonl"),
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Backfilled wiring (`open-interpretability wiring-backfill`) for records without "
+    "their own; skipped if missing.",
+)
+@click.option(
+    "--corpus-sources",
+    "source_paths",
+    multiple=True,
+    default=[Path(name) for name in COMMITTED_SOURCES],
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Corpus sidecars that give the source of records which scanned their corpus "
+    "(matched by SHA-256) without recording one, shown as inferred; skipped if missing.",
+)
+@click.option(
+    "--alignment",
+    "alignment_path",
+    default=Path("docs/analyses/dictionary-alignment.jsonl"),
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Dictionary latents aligned with each neuron pole (`open-interpretability crossref`); "
+    "skipped if missing.",
+)
+@click.option(
+    "--sisters",
+    "sisters_path",
+    default=SISTERS_PATH,
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Each VERIFIED pole's matches in other models (`open-interpretability sisters`); "
+    "skipped if missing.",
+)
+@click.option(
+    "--coverage",
+    "coverage_path",
+    default=Path("docs/analyses/registry-coverage.jsonl"),
+    show_default=True,
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Registry coverage rows (`open-interpretability coverage`) for the home page; "
+    "skipped if missing.",
+)
+@click.option(
+    "--papers",
+    "papers_dir",
+    default=Path("papers"),
+    show_default=True,
+    type=click.Path(file_okay=False, path_type=Path),
+    help="The papers of the Research section; the section is empty if missing.",
+)
+@click.option(
+    "--jobs",
+    type=click.IntRange(min=1),
+    default=None,
+    help="Worker processes that render the unit pages. Default: one per CPU. 1 renders in "
+    "this process.",
+)
+def build(  # noqa: PLR0913 (one option per input file)
+    wiki_dir,
+    output_dir,
+    wiring_path,
+    source_paths,
+    alignment_path,
+    sisters_path,
+    coverage_path,
+    papers_dir,
+    jobs,
+):
+    """Compile every archived experiment into static pages and data/registry.parquet."""
+    from openinterp.builder.compiler import compile_site
+    from openinterp.builder.coverage import load_rows as load_coverage
+    from openinterp.builder.model_facts import load_model_facts
+    from openinterp.builder.parallel import default_jobs
+    from openinterp.builder.users import load_users
+    from openinterp.builder.wiki import load_archive
+    from openinterp.core.draws import load_draws, load_stops
+    from openinterp.core.errata import load_errata
+    from openinterp.core.project import project_links
+    from openinterp.core.takedowns import activate, load_takedowns
+    from openinterp.corpus import load_sources
+    from openinterp.crossref import load_rows as load_alignment
+    from openinterp.planner import configured_models
+    from openinterp.sisters import load_rows as load_sisters
+    from openinterp.wiring_backfill import load_rows
+
+    activate(load_takedowns(wiki_dir))  # before any text is scrubbed; workers fork after this
+    report = compile_site(
+        load_archive(wiki_dir),
+        output_dir,
+        wiring_rows=load_rows(wiring_path),
+        draws=load_draws(wiki_dir),
+        corpus_sources=load_sources(source_paths),
+        alignment_rows=load_alignment(alignment_path),
+        sister_rows=load_sisters(sisters_path),
+        papers_dir=papers_dir,
+        stops=load_stops(wiki_dir),
+        model_facts=load_model_facts(configured_models()),  # config.json only, no weights
+        jobs=jobs or default_jobs(),
+        errata=load_errata(wiki_dir),
+        users=load_users(project_links()["repo_url"], wiki_dir),
+        coverage_rows=load_coverage(coverage_path),
+    )
+    click.echo(
+        f"Published {report.experiments_published} experiments: {len(report.pages)} pages, "
+        f"{report.parquet} → {output_dir}"
+    )

openinterp/cli_check.py

@@ -0,0 +1,51 @@
+"""The ``check`` command: registry consistency, for CI and before a push."""
+
+from __future__ import annotations
+
+import click
+
+from openinterp.cli import cli, wiki_dir_option
+
+
+@cli.command()
+@wiki_dir_option()
+@click.option(
+    "--base-ref",
+    help="Git ref to diff against (e.g. origin/main). Enforces write-once pages, and checks "
+    "only the branch's changes unless it also changes files outside the wiki.",
+)
+def check(wiki_dir, base_ref):
+    """Check that every posted experiment has a valid record and wiki page (CI gate)."""
+    import os
+    import subprocess
+
+    from openinterp.checks import check_registry
+
+    try:
+        report = check_registry(wiki_dir, base_ref=base_ref)
+    except subprocess.CalledProcessError as error:
+        raise click.ClickException(
+            f"can't diff against {base_ref} ({error.stderr.strip()}). "
+            "Fetch the ref, or run `open-interpretability check` to check every record."
+        ) from error
+    in_actions = os.environ.get("GITHUB_ACTIONS") == "true"
+    for problem in report.problems:
+        if in_actions:
+            click.echo(f"::error file={problem.path}::{problem.message}")
+        else:
+            click.echo(f"error: {problem}", err=True)
+
+    summary = (
+        f"Checked {report.records} experiment records and {report.pages} experiment pages"
+        + "".join(f" and {n} {kind} records" for kind, n in report.extra_counts.items() if n)
+        + (f" changed since {base_ref}" if report.changed_only else "")
+        + ": "
+        + ("all consistent." if report.ok else f"{len(report.problems)} problem(s).")
+    )
+    click.echo(summary)
+    if in_actions and os.environ.get("GITHUB_STEP_SUMMARY"):
+        with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh:
+            fh.write(f"### Experiment check\n\n{summary}\n\n")
+            fh.writelines(f"- `{p.path}`: {p.message}\n" for p in report.problems)
+    if not report.ok:
+        raise SystemExit(1)

openinterp/cli_concepts.py

@@ -0,0 +1,76 @@
+"""``open-interpretability concepts``: a labelled concept set from an annotated dataset.
+
+See :mod:`openinterp.concepts` for the presets and how labels become spans.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from openinterp.cli import cli
+from openinterp.concepts import CONCEPT_PRESETS
+
+
+@cli.command()
+@click.option("--dataset", "preset", required=True, type=click.Choice(list(CONCEPT_PRESETS)))
+@click.option(
+    "--label",
+    help="The label to mark (PER, NOUN, nsubj, fr, Sports). Omit it to list the split's "
+    "labels and how many words or documents carry each.",
+)
+@click.option("--split", help="Dataset split. Default: the preset's held-out one.")
+@click.option("--subset", help="Universal Dependencies treebank (default en_ewt).")
+@click.option("--column", help="Read labels from this column instead (e.g. UD's xpos).")
+@click.option("--docs", "documents", default=1000, show_default=True, type=click.IntRange(1))
+@click.option("--seed", default=0, show_default=True, type=int)
+@click.option(
+    "--max-chars",
+    default=3000,
+    show_default=True,
+    type=click.IntRange(1),
+    help="Join sentences into documents of at most this many characters.",
+)
+@click.option("--out", type=click.Path(path_type=Path), help="Concept set JSONL.")
+def concepts(preset, label, split, subset, column, documents, seed, max_chars, out):
+    """Write a concept set from human-annotated labels.
+
+    The set is JSON lines of {"text": ..., "spans": [[start, end], ...]}. Use it for
+    `open-interpretability probe-compare --train/--test`, sparse-probing searches and reviewers'
+    positive holdouts.
+
+    Word tags (conll2003-*, ud-*) mark each tagged word; document classes
+    (language-identification, ag_news) mark whole documents, half of them of the class.
+    Also writes <name>.source.json, which pins the dataset commit, split and licence.
+    """
+    from openinterp.concepts import build_concept, load_split
+
+    if label is None:
+        loaded = load_split(preset, split=split, subset=subset, column=column, max_chars=max_chars)
+        click.echo(
+            f"{loaded.dataset}@{loaded.revision[:8]} {loaded.files}: {len(loaded.docs)} docs"
+        )
+        for tag, n in loaded.counts().most_common():
+            click.echo(f"{n:>8,}  {tag}")
+        return
+    if out is None:
+        raise click.UsageError("--out is required with --label.")
+    try:
+        source = build_concept(
+            preset,
+            label,
+            out,
+            split=split,
+            subset=subset,
+            column=column,
+            documents=documents,
+            seed=seed,
+            max_chars=max_chars,
+        )
+    except ValueError as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Wrote {out}: {source}")
+    if source.skipped_sentences:
+        click.echo(f"warning: {source.skipped_sentences} sentences' words not found", err=True)
+    click.echo(f"Licence: {source.license or 'not stated on the dataset card'}")

openinterp/cli_corpus.py

@@ -0,0 +1,252 @@
+"""``open-interpretability corpus``: build a scan corpus from a Hugging Face dataset."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from click.core import ParameterSource
+
+from openinterp.blends import BLEND_ALIASES, BLENDS, DEFAULT_DATASET
+from openinterp.cli import cli
+from openinterp.corpus import LAYOUTS, PRESETS
+
+
+@cli.command()
+@click.option(
+    "--dataset",
+    default=DEFAULT_DATASET,
+    show_default=True,
+    help=f"A blend of presets ({', '.join([*BLEND_ALIASES, *BLENDS])}; '{DEFAULT_DATASET}' is "
+    f"the latest), a preset ({', '.join(PRESETS)}) or a Hugging Face dataset ID (then pass "
+    "--files).",
+)
+@click.option(
+    "--files",
+    help="Glob of Parquet files within the dataset; comma-separate several to take an equal "
+    "share of documents from each. Default: the preset's.",
+)
+@click.option(
+    "--text-column",
+    help="Column holding the text; comma-separate several to join them into one document. "
+    "Default: the preset's, or 'text'.",
+)
+@click.option(
+    "--revision",
+    help="Dataset branch, tag or commit. Default: the preset's, or main; pinned on use.",
+)
+@click.option(
+    "--layout",
+    type=click.Choice(list(LAYOUTS)),
+    help="Join rows into documents this way, e.g. one WikiText row per paragraph. Default: "
+    "the preset's, or one document per row.",
+)
+@click.option("--docs", "documents", default=500, show_default=True, type=click.IntRange(1))
+@click.option("--seed", default=0, show_default=True, type=int)
+@click.option(
+    "--max-chars",
+    default=3000,
+    show_default=True,
+    type=click.IntRange(1),
+    help="Cut each document to this many characters.",
+)
+@click.option(
+    "--docs-per-row-group",
+    type=click.IntRange(1),
+    help="Documents taken from each Parquet row group read; lower spreads the sample wider. "
+    "Default: the preset's, or 20.",
+)
+@click.option(
+    "--first",
+    is_flag=True,
+    help="Take the dataset's first --docs documents in file order, cut but not stripped, "
+    "instead of a seeded sample.",
+)
+@click.option(
+    "--ascii",
+    "ascii_",
+    is_flag=True,
+    help="Escape non-ASCII characters in the JSONL. With --first, rebuilds the registry's "
+    "corpus from pile-10k (README).",
+)
+@click.option(
+    "--lang-pattern",
+    help="Regular expression whose group, matched in each --files glob, is the language its "
+    "documents record as 'lang' (e.g. '^\\d+\\.(\\w+)/' for Wikipedia). Default: the preset's.",
+)
+@click.option(
+    "--detect-lang",
+    is_flag=True,
+    help="Detect each document's language (fastText lid.176, downloaded once) and record "
+    "it as 'lang' where the detector is sure, for datasets that don't say. The source "
+    "sidecar records the detector. Replaces a preset's exact languages; not with --lang-pattern.",
+)
+@click.option(
+    "--detect-code",
+    is_flag=True,
+    help="Detect each document's programming language (Magika) and record it as "
+    "'code_language' where the detector is sure, for datasets that don't say (e.g. the "
+    "Pile's GitHub files). Prose stays unlabelled. The source sidecar records the "
+    "detector. Not with a preset that gives exact programming languages.",
+)
+@click.option(
+    "--chat-template",
+    metavar="MODEL",
+    help="Write each conversation in --text-column (a list of role/content messages) with "
+    "this model's chat template, pinned to its commit. Default: the preset's ("
+    + ", ".join(f"{name}: {p.chat_template}" for name, p in PRESETS.items() if p.chat_template)
+    + "), or none.",
+)
+@click.option(
+    "--same-as",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="Rebuild the corpus this record scanned, at its pinned dataset commits, and check "
+    "its SHA-256. It sets the source, so only --out goes with it.",
+)
+@click.option(
+    "--out", default=Path("corpus.jsonl"), show_default=True, type=click.Path(path_type=Path)
+)
+@click.pass_context
+def corpus(
+    ctx,
+    dataset,
+    files,
+    text_column,
+    revision,
+    layout,
+    documents,
+    seed,
+    max_chars,
+    docs_per_row_group,
+    first,
+    ascii_,
+    lang_pattern,
+    detect_lang,
+    detect_code,
+    chat_template,
+    same_as,
+    out,
+):
+    """Sample a scan corpus from a Hugging Face dataset, without downloading all of it.
+
+    Writes <out> (one {"text": ...} per line) and <name>.source.json next to it, which
+    pins the dataset commit, files, seed and licence. `open-interpretability audit --corpus <out>`
+    records that source in the submission's provenance.
+    """
+    from openinterp.corpus import build_from_hub
+
+    if same_as:
+        return _same_as(ctx, same_as, out)
+    if dataset in BLENDS or dataset in BLEND_ALIASES:
+        per_dataset = {
+            "--files": files, "--text-column": text_column, "--revision": revision,
+            "--layout": layout, "--docs-per-row-group": docs_per_row_group, "--first": first,
+            "--lang-pattern": lang_pattern, "--detect-lang": detect_lang,
+            "--detect-code": detect_code, "--chat-template": chat_template,
+        }  # fmt: skip
+        return _blend(dataset, per_dataset, out, documents, seed, max_chars, ascii_)
+    preset = PRESETS.get(dataset)
+    _check_dataset(dataset, preset, files, lang_pattern, detect_lang, detect_code)
+    try:
+        source = build_from_hub(
+            preset.dataset if preset else dataset,
+            out,
+            files=files or preset.files,
+            text_column=text_column or (preset.text_column if preset else "text"),
+            revision=revision or (preset.revision if preset else None),
+            layout=layout or (preset.layout if preset else None),
+            documents=documents,
+            seed=seed,
+            max_chars=max_chars,
+            docs_per_row_group=docs_per_row_group or (preset.docs_per_row_group if preset else 20),
+            first=first,
+            ascii=ascii_,
+            lang_pattern=None if detect_lang else lang_pattern or (preset and preset.lang_pattern),
+            detect_lang=detect_lang,
+            code_language_column=preset.code_language_column if preset else None,
+            detect_code=detect_code,
+            chat_template=chat_template or (preset.chat_template if preset else None),
+        )
+    except ValueError as error:  # e.g. a model without a chat template
+        raise click.ClickException(str(error)) from error
+    _report(source, out, documents)
+
+
+def _check_dataset(dataset, preset, files, lang_pattern, detect_lang, detect_code) -> None:
+    """Raise ``click.UsageError`` if the options can't sample ``dataset``."""
+    if detect_lang and lang_pattern:
+        raise click.UsageError("--detect-lang and --lang-pattern both set the languages.")
+    if detect_code and preset and preset.code_language_column:
+        raise click.UsageError(f"{dataset!r} has exact programming languages: no --detect-code.")
+    if preset is None and not files:
+        raise click.UsageError(f"--files is required for {dataset!r}, which is not a preset.")
+
+
+@cli.command("corpus-cache")
+@click.option("--clear", is_flag=True, help="Delete everything cached.")
+def corpus_cache(clear):
+    """Show the size of (or clear) the cache of Parquet data that `corpus` reads.
+
+    \b
+    Location: $OPENINTERP_CORPUS_CACHE (\"off\" disables it), else
+    $XDG_CACHE_HOME/openinterp/corpora or ~/.cache/openinterp/corpora.
+    Size cap: $OPENINTERP_CORPUS_CACHE_GB (default 5); least recently used go first.
+    """
+    from openinterp.corpus_cache import ParquetCache
+
+    cache = ParquetCache.default()
+    if cache is None:
+        click.echo("The corpus cache is off ($OPENINTERP_CORPUS_CACHE).")
+        return
+    if clear:
+        click.echo(f"Deleted {cache.clear()} entries from {cache.root}")
+        return
+    click.echo(
+        f"{cache.root}: {len(cache.entries())} entries, "
+        f"{cache.size() / 1e9:.2f} of {cache.max_bytes / 1e9:.0f} GB"
+    )
+
+
+def _same_as(ctx, record: Path, out: Path) -> None:
+    from openinterp.cli_review import load_submission
+    from openinterp.corpus_rebuild import rebuild, scanned_source
+
+    given = [
+        name
+        for name in ctx.params
+        if name not in ("same_as", "out")
+        and ctx.get_parameter_source(name) != ParameterSource.DEFAULT
+    ]
+    if given:
+        raise click.UsageError(f"--same-as sets the source, so not with: {', '.join(given)}.")
+    try:
+        source = rebuild(scanned_source(load_submission(record)), out)
+    except ValueError as error:
+        raise click.ClickException(f"{record}: {error}") from error
+    _report(source, out, source.documents)
+
+
+def _blend(dataset, per_dataset: dict, out: Path, documents, seed, max_chars, ascii_) -> None:
+    from openinterp.blends import build_blend
+
+    if given := [name for name, value in per_dataset.items() if value]:
+        raise click.UsageError(
+            f"{', '.join(given)}: set per preset, so not with the blend {dataset!r}."
+        )
+    source = build_blend(
+        dataset, out, documents=documents, seed=seed, max_chars=max_chars, ascii=ascii_
+    )
+    _report(source, out, documents)
+    for part in source.components:
+        click.echo(f"  {part.documents} from {part}")
+
+
+def _report(source, out: Path, documents: int) -> None:
+    click.echo(f"Wrote {source.documents} documents to {out}: {source}")
+    if source.lang_detector:
+        click.echo(f"Languages detected by {source.lang_detector}", err=True)
+    if source.code_detector:
+        click.echo(f"Programming languages detected by {source.code_detector}", err=True)
+    if source.documents < documents:
+        click.echo(f"warning: only {source.documents} of {documents} documents found", err=True)
+    click.echo(f"Licence: {source.license or 'not stated on the dataset card'}")