commonground-engine 0.2.0__py3-none-any.whl

commonground/__main__.py

@@ -0,0 +1,499 @@
+"""Command line: ``python -m commonground <command>``.
+
+    check     decide one argument given as formulas
+    analyze   conflicts, concessions and justifications for a debate file
+    extract   extract the statements of text files, with verbatim spans
+    camp      per-camp report: statements -> axioms -> load -> certificates
+    crosscamp common ground + fault lines for two or more camp records
+    co2       the bundled CO2 golden example, end to end (headless)
+    folio     run the FOLIO benchmark (reference formalizations, or a language model)
+    compare   paired comparison of two language-model runs, as JSON
+    report    the same comparison as a Markdown summary
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from . import argmap, bench, folio, lean
+from .fol import Not, ParseError, parse
+from .verdict import decide
+
+
+def _check(args) -> int:
+    try:
+        premises = [parse(p) for p in args.premise]
+        conclusion = parse(args.conclusion)
+    except ParseError as exc:
+        print(f"not a formula: {exc}", file=sys.stderr)
+        return 2
+    verdict = decide(premises, conclusion, args.timeout_ms)
+    out = verdict.to_json()
+    if args.lean and verdict.label in ("True", "False"):
+        lean_bin = lean.find_lean()
+        if lean_bin is None:
+            out["lean"] = f"no Lean {lean.TOOLCHAIN} found (set LEAN_BIN)"
+        else:
+            goal = conclusion if verdict.label == "True" else Not(conclusion)
+            certificate = lean.certify([(premises, goal)], lean_bin)[0]
+            out["lean"] = {"certified": certificate.accepted, "axioms": certificate.axioms,
+                           "reason": certificate.reason}
+    print(json.dumps(out, indent=2, ensure_ascii=False))
+    return 0
+
+
+TS_HEADER = """// Generated by `python -m commonground analyze {source} --out {target}`.
+// Do not edit: engine/tests/test_generated.py fails when this file is out of date.
+
+export interface StatementGroup {{
+  statements: string[];
+  positions: string[];
+}}
+
+export interface DebateAnalysis {{
+  /** SHA-256 of the statements the analysis was computed from. */
+  input_hash: string;
+  premise_ids: string[];
+  positions: string[];
+  /** All premises of all positions have a common model. */
+  positions_compatible: boolean;
+  compatibility_witness: {{ domain_size: number; engine: string; rechecked_by_independent_evaluator: boolean }} | null;
+  /** Minimal sets of premises that cannot all hold. Empty when the positions are compatible. */
+  conflicts: StatementGroup[];
+  /** Smallest sets of premises whose withdrawal restores consistency. */
+  minimal_concessions: StatementGroup[];
+  /** Premises that belong to every maximal consistent set. */
+  uncontested: string[];
+  position_consistent: Record<string, boolean>;
+  conclusions: Record<string, {{
+    owner: string;
+    from_all_premises: string;
+    from_each_position: Record<string, string>;
+    /** Every minimal set of premises that entails the conclusion. */
+    justifications: string[][];
+  }}>;
+  statements_without_formula: string[];
+}}
+
+export const analysis: DebateAnalysis = {body};
+"""
+
+
+def render_analysis(debate: str, out: str | None, timeout_ms: int = 5000) -> str:
+    _, statements = argmap.load_debate(debate)
+    body = json.dumps(argmap.analyze(statements, timeout_ms), indent=2, ensure_ascii=False)
+    if out and out.endswith(".ts"):
+        return TS_HEADER.format(source=Path(debate).as_posix(), target=Path(out).as_posix(), body=body)
+    return body + "\n"
+
+
+def _analyze(args) -> int:
+    text = render_analysis(args.debate, args.out, args.timeout_ms)
+    if args.out:
+        Path(args.out).write_text(text, encoding="utf-8")
+    else:
+        print(text, end="")
+    return 0
+
+
+def _folio(args) -> int:
+    out_dir = Path(args.out)
+    if args.mode == "reference":
+        lean_bin = lean.find_lean() if args.lean else None
+        if args.lean and lean_bin is None:
+            print(f"no Lean {lean.TOOLCHAIN} found (set LEAN_BIN)", file=sys.stderr)
+            return 2
+        summary = bench.reference(folio.fetch(Path(args.data)), out_dir, lean_bin, args.timeout_ms)
+    elif args.mode == "abduction":
+        lean_bin = lean.find_lean() if args.lean else None
+        if args.lean and lean_bin is None:
+            print(f"no Lean {lean.TOOLCHAIN} found (set LEAN_BIN)", file=sys.stderr)
+            return 2
+        data_file = folio.fetch(Path(args.data))
+        try:
+            summary = bench.abduction(data_file, out_dir, args.limit, args.timeout_ms, lean_bin,
+                                      args.lam, args.max_checks)
+        except bench.ConfigurationMismatch as exc:
+            print(exc, file=sys.stderr)
+            return 2
+    else:
+        from .formalize import OpenAICompatible, ProviderError
+        key = os.environ.get(args.api_key_env, "")
+        if not (args.base_url and args.model and key):
+            print(f"--base-url, --model and the {args.api_key_env} environment variable are required", file=sys.stderr)
+            return 2
+        try:
+            client = OpenAICompatible(args.base_url, args.model, key, json_mode=not args.no_json_mode,
+                                      temperature=None if args.no_temperature else args.temperature,
+                                      max_tokens=args.max_tokens)
+        except ValueError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        data_file = folio.fetch(Path(args.data))
+        try:
+            summary = bench.with_model(data_file, out_dir, client, args.mode, args.limit, args.timeout_ms)
+        except bench.ConfigurationMismatch as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        except ProviderError as exc:
+            # The run stops instead of scoring an outage as wrong answers; the same command resumes it.
+            print(f"stopped: {exc}\nfinished examples are kept in {out_dir / 'runs.jsonl'}; "
+                  "run the same command again to resume", file=sys.stderr)
+            if exc.status in (400, 422):
+                print("if the provider's message names response_format or temperature, add --no-json-mode "
+                      "or --no-temperature (in a new --out directory: the settings are part of the run)",
+                      file=sys.stderr)
+            return 3
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+    return 0
+
+
+def _load_plausibilities(path: str | None) -> dict:
+    if not path:
+        return {}
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError("the plausibilities file must be a JSON object mapping FOL text to a number in [0, 1] "
+                         "or a two-element range [min, max]")
+    for key, value in data.items():
+        ok = (isinstance(value, (int, float)) and not isinstance(value, bool)) or \
+             (isinstance(value, list) and len(value) == 2 and all(
+                 isinstance(v, (int, float)) and not isinstance(v, bool) for v in value))
+        if not ok:
+            raise ValueError(f"the plausibility of {key!r} must be a number in [0, 1] or a [min, max] range, "
+                             f"got {value!r}")
+    return data
+
+
+def _lean_for(args) -> tuple[str | None, bool]:
+    """(lean executable or None, ok). Asking for kernel certificates without a Lean of the
+    pinned version is an error, not a silent fall-back to solver-only badges."""
+    if not args.lean:
+        return None, True
+    lean_bin = lean.find_lean()
+    if lean_bin is None:
+        print(f"no Lean {lean.TOOLCHAIN} found (set LEAN_BIN), or drop --lean for solver-checked verdicts",
+              file=sys.stderr)
+    return lean_bin, lean_bin is not None
+
+
+def _camp_dir(out: Path, camp: str) -> Path:
+    """<out>/<camp>; a camp name that is not a plain directory name is refused, so a
+    record cannot make the report land outside the output directory."""
+    if (camp != camp.strip() or camp in ("", ".", "..") or camp.endswith(".")
+            or any(ch in '/\\:*?"<>|' or ord(ch) < 32 for ch in camp)):
+        raise ValueError(f"camp name {camp!r} cannot be used as a directory name")
+    return out / camp
+
+
+def _write_camp_report(directory: Path, report: dict) -> None:
+    from . import abduce
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    (directory / "report.md").write_text(abduce.render_markdown(report), encoding="utf-8")
+
+
+def _camp(args) -> int:
+    from . import abduce
+    from .argmap import TooMany
+    try:
+        record = abduce.load_record(args.record)
+        candidates = abduce.load_candidates(args.candidates)
+        plausibilities = _load_plausibilities(args.plausibilities)
+    except (OSError, ValueError) as exc:
+        print(f"cannot read the inputs: {exc}", file=sys.stderr)
+        return 2
+    lean_bin, ok = _lean_for(args)
+    if not ok:
+        return 2
+    try:
+        report = abduce.report_camp(record, abduce.ScriptedSource(candidates), plausibilities,
+                                    args.lam, args.timeout_ms, lean_bin, args.max_checks)
+    except TooMany as exc:
+        print(f"input too large for the documented scope: {exc}", file=sys.stderr)
+        return 2
+    except ValueError as exc:
+        print(f"cannot run the analysis: {exc}", file=sys.stderr)
+        return 2
+    _write_camp_report(Path(args.out), report)
+    print(json.dumps({"out": str(Path(args.out)), "camp": report["camp"],
+                      "selected_systems": len(report["systems"]),
+                      "search_complete": report["search_complete"],
+                      "abstention": report["abstention"]}, ensure_ascii=False))
+    return 0
+
+
+def _crosscamp_summary(out: Path, result: dict, reports: list[dict]) -> str:
+    n_mus = sum(len(p.get("mus") or []) for p in result["fault_lines"]["per_system_pair"])
+    return json.dumps({"out": str(out), "camps": result["camps"],
+                       "stated_agreed": len(result["common_ground"]["stated"]["agreed"]),
+                       "fault_lines_found": n_mus,
+                       "search_complete": {r["camp"]: r["search_complete"] for r in reports}},
+                      ensure_ascii=False)
+
+
+def _crosscamp(args) -> int:
+    from . import abduce, crosscamp
+    from .argmap import TooMany
+    if len(args.records) < 2:
+        print("crosscamp needs at least two camp records", file=sys.stderr)
+        return 2
+    if len(args.candidates) != len(args.records):
+        print("--candidates once per record, in the same order", file=sys.stderr)
+        return 2
+    out = Path(args.out)
+    try:
+        plausibilities = _load_plausibilities(args.plausibilities)
+        camps = []
+        for rec_path, cand_path in zip(args.records, args.candidates):
+            record = abduce.load_record(rec_path)
+            camps.append((record, abduce.load_candidates(cand_path), _camp_dir(out, record.camp)))
+        names = [record.camp for record, _, _ in camps]
+        if len(set(names)) != len(names):
+            raise ValueError(f"camp names must be unique, got {names}")
+    except (OSError, ValueError) as exc:
+        print(f"cannot read the inputs: {exc}", file=sys.stderr)
+        return 2
+    lean_bin, ok = _lean_for(args)
+    if not ok:
+        return 2
+    out.mkdir(parents=True, exist_ok=True)
+    camp_objs = []
+    try:
+        for record, candidates, camp_dir in camps:
+            report = abduce.report_camp(record, abduce.ScriptedSource(candidates), plausibilities,
+                                        args.lam, args.timeout_ms, lean_bin, args.max_checks)
+            _write_camp_report(camp_dir, report)
+            camp_objs.append(crosscamp.Camp(record, report))
+        result = crosscamp.analyze(camp_objs, args.timeout_ms, lean_bin)
+    except TooMany as exc:
+        print(f"input too large for the documented scope: {exc}; per-camp reports were written",
+              file=sys.stderr)
+        return 2
+    except ValueError as exc:
+        print(f"cannot run the analysis: {exc}", file=sys.stderr)
+        return 2
+    (out / "crosscamp.json").write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
+    (out / "crosscamp.md").write_text(crosscamp.render_markdown(result), encoding="utf-8")
+    print(_crosscamp_summary(out, result, [c.report for c in camp_objs]))
+    return 0
+
+
+def _camp_from_corpus(camp_id, statements, golden_camp):
+    """Build a Record plus scripted candidates from the CO2 corpus files. The corpus ships
+    statement texts, not whole booklet pages: the document is the camp's statements joined
+    by newline, spans located in it, so the verbatim-span discipline still holds."""
+    from . import abduce, fol
+    texts = {s["id"]: s["text"] for s in statements if s["camp"] == camp_id}
+    fol_by_id = {s["id"]: s["fol"] for s in golden_camp["statements"]}
+    doc_id = f"doc-{camp_id}"
+    parts, spans = [], {}
+    offset = 0
+    for s in golden_camp["statements"]:
+        t = texts[s["id"]]
+        parts.append(t)
+        spans[s["id"]] = (offset, offset + len(t))
+        offset += len(t) + 1
+    record = abduce.Record(camp_id, [abduce.Document(doc_id, "\n".join(parts))],
+                           [abduce.RecordStatement(s["id"], "endorsed", fol_by_id[s["id"]], doc_id,
+                                                   texts[s["id"]], spans[s["id"]])
+                            for s in golden_camp["statements"]])
+    cands = [abduce.Candidate(c["fol"], [(doc_id, texts[ev], spans[ev]) for ev in c["evidence"]])
+             for c in golden_camp["candidate_axioms"]]
+    plaus = {fol.to_text(fol.parse(c["fol"])): c["plausibility"] for c in golden_camp["candidate_axioms"]}
+    return record, cands, plaus
+
+
+def _co2(args) -> int:
+    from . import abduce, crosscamp
+    from .argmap import TooMany
+    ex = Path(args.dir)
+    try:
+        statements = json.loads((ex / "statements.json").read_text(encoding="utf-8"))["statements"]
+        golden = json.loads((ex / "golden.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError, KeyError) as exc:
+        print(f"cannot read the corpus in {ex}: {exc}", file=sys.stderr)
+        return 2
+    lean_bin, ok = _lean_for(args)
+    if not ok:
+        return 2
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    camps = []
+    try:
+        for camp_id, g in golden["camps"].items():
+            record, cands, plaus = _camp_from_corpus(camp_id, statements, g)
+            report = abduce.report_camp(record, abduce.ScriptedSource(cands), plaus, args.lam,
+                                        args.timeout_ms, lean_bin, args.max_checks)
+            _write_camp_report(_camp_dir(out, camp_id), report)
+            camps.append(crosscamp.Camp(record, report))
+        result = crosscamp.analyze(camps, args.timeout_ms, lean_bin)
+    except TooMany as exc:
+        print(f"input too large for the documented scope: {exc}; per-camp reports were written",
+              file=sys.stderr)
+        return 2
+    except (ValueError, KeyError) as exc:
+        print(f"cannot run the analysis on the corpus in {ex}: {exc}", file=sys.stderr)
+        return 2
+    (out / "crosscamp.json").write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
+    (out / "crosscamp.md").write_text(crosscamp.render_markdown(result), encoding="utf-8")
+    print(_crosscamp_summary(out, result, [c.report for c in camps]))
+    return 0
+
+
+def _extract(args) -> int:
+    from . import extract as extract_mod
+    from .formalize import OpenAICompatible, ProviderError
+    key = os.environ.get(args.api_key_env, "")
+    if not (args.base_url and args.model and key):
+        print(f"--base-url, --model and the {args.api_key_env} environment variable are required", file=sys.stderr)
+        return 2
+    try:
+        client = OpenAICompatible(args.base_url, args.model, key, json_mode=not args.no_json_mode,
+                                  temperature=None if args.no_temperature else args.temperature,
+                                  max_tokens=args.max_tokens)
+    except ValueError as exc:
+        print(exc, file=sys.stderr)
+        return 2
+    try:
+        documents = [(path, Path(path).read_text(encoding="utf-8")) for path in args.files]
+    except OSError as exc:
+        print(f"cannot read a document: {exc}", file=sys.stderr)
+        return 2
+    out_dir = Path(args.out)
+    try:
+        summary = extract_mod.run(client, documents, out_dir)
+    except ValueError as exc:  # a configuration mismatch, or a document that fails validation
+        print(exc, file=sys.stderr)
+        return 2
+    except ProviderError as exc:
+        # The run stops instead of scoring an outage; the same command resumes it.
+        print(f"stopped: {exc}\nfinished documents are kept in {out_dir / 'runs.jsonl'}; "
+              "run the same command again to resume", file=sys.stderr)
+        return 3
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+    return 0
+
+
+def _compare(args) -> int:
+    try:
+        if args.command == "report":
+            print(bench.report(Path(args.first), Path(args.second)), end="")
+        else:
+            print(json.dumps(bench.compare(Path(args.first), Path(args.second)), indent=2, ensure_ascii=False))
+    except (OSError, ValueError, KeyError) as exc:
+        print(f"cannot compare: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="commonground", description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--timeout-ms", type=int, default=5000, help="solver limit per satisfiability question")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    check = commands.add_parser("check", help="decide one argument")
+    check.add_argument("--premise", action="append", default=[], help="a formula; repeat for each premise")
+    check.add_argument("--conclusion", required=True)
+    check.add_argument("--lean", action="store_true", help="also ask the Lean kernel to certify the verdict")
+    check.set_defaults(run=_check)
+
+    analyze = commands.add_parser("analyze", help="analyse a debate file")
+    analyze.add_argument("debate")
+    analyze.add_argument("--out")
+    analyze.set_defaults(run=_analyze)
+
+    camp = commands.add_parser("camp", help="per-camp report: statements -> axioms -> load -> certificates")
+    camp.add_argument("record", help="camp record JSON (camp, documents, statements with stances/spans)")
+    camp.add_argument("--candidates", required=True,
+                      help="JSON file of scripted unstated candidates: {\"candidates\": [{\"fol\", \"quotes\": [...]}]}")
+    camp.add_argument("--plausibilities", help="optional JSON object mapping FOL text to a plausibility estimate: a number in [0, 1] or a [min, max] range")
+    camp.add_argument("--lam", type=float, default=0.3, help="cost weight of one axiom (default 0.3)")
+    camp.add_argument("--lean", action="store_true", help="also ask the Lean kernel to certify sufficiency")
+    camp.add_argument("--max-checks", type=int, default=12000,
+                      help="budget of certification checks (default 12000, enough to finish the CO2 "
+                           "`yes` search, AUDIT-4); a search it stops is reported as incomplete, never "
+                           "as cost-minimal")
+    camp.add_argument("--out", required=True, help="output directory: report.json and report.md")
+    camp.set_defaults(run=_camp)
+
+    xc = commands.add_parser("crosscamp", help="cross-camp report: common ground + fault lines for 2+ camps")
+    xc.add_argument("records", nargs="+", help="camp record JSONs (two or more)")
+    xc.add_argument("--candidates", nargs="+", required=True,
+                    help="one scripted-candidates JSON per record, in the same order")
+    xc.add_argument("--plausibilities", help="optional JSON object mapping FOL text to a plausibility estimate: a number in [0, 1] or a [min, max] range")
+    xc.add_argument("--lam", type=float, default=0.3, help="cost weight of one axiom (default 0.3)")
+    xc.add_argument("--lean", action="store_true", help="also ask the Lean kernel to certify sufficiency")
+    xc.add_argument("--timeout-ms", type=int, default=5000, help="solver limit per satisfiability question")
+    xc.add_argument("--max-checks", type=int, default=12000,
+                    help="budget of certification checks per camp (default 12000); a search it stops is "
+                         "reported as incomplete, never as cost-minimal")
+    xc.add_argument("--out", required=True,
+                    help="output directory: <camp>/report.{json,md} per camp, plus crosscamp.{json,md}")
+    xc.set_defaults(run=_crosscamp)
+
+    co2 = commands.add_parser("co2", help="run the bundled CO2 golden example end to end (headless)")
+    co2.add_argument("--dir",
+                     default=str(Path(__file__).resolve().parents[2] / "examples" / "co2-2021"),
+                     help="corpus directory holding statements.json and golden.json")
+    co2.add_argument("--lam", type=float, default=0.3, help="cost weight of one axiom (default 0.3)")
+    co2.add_argument("--lean", action="store_true", help="also ask the Lean kernel to certify sufficiency")
+    co2.add_argument("--timeout-ms", type=int, default=5000, help="solver limit per satisfiability question")
+    co2.add_argument("--max-checks", type=int, default=12000,
+                    help="cap on certification checks per camp (default 12000, enough for the CO2 `yes` "
+                         "search; see AUDIT.md AUDIT-4)")
+    co2.add_argument("--out", required=True,
+                     help="output directory: <camp>/report.{json,md} per camp, plus crosscamp.{json,md}")
+    co2.set_defaults(run=_co2)
+
+    ex = commands.add_parser("extract", help="extract the statements of text files, with verbatim spans")
+    ex.add_argument("files", nargs="+", help="text files; each file is one document, id = the path as given")
+    ex.add_argument("--out", required=True, help="output directory: runs.jsonl, config.json, summary.json")
+    ex.add_argument("--base-url")
+    ex.add_argument("--model")
+    ex.add_argument("--api-key-env", default="LLM_API_KEY", help="name of the variable holding the key")
+    ex.add_argument("--no-json-mode", action="store_true",
+                    help="do not send response_format=json_object (for providers that reject it)")
+    ex.add_argument("--temperature", type=float, default=0.0, help="sampling temperature (default 0)")
+    ex.add_argument("--no-temperature", action="store_true",
+                    help="send no temperature (for models that accept only their default)")
+    ex.add_argument("--max-tokens", type=int, default=8000, help="reply length limit (default 8000)")
+    ex.set_defaults(run=_extract)
+
+    fol = commands.add_parser("folio", help="run the FOLIO benchmark")
+    fol.add_argument("--data", default="data/FOLIO", help="where the dataset repository is (cloned if absent)")
+    fol.add_argument("--out", required=True)
+    fol.add_argument("--mode", choices=("reference", "llm", "direct", "abduction"), default="reference")
+    fol.add_argument("--lam", type=float, default=0.3, help="abduction mode: cost weight of one axiom (default 0.3)")
+    fol.add_argument("--max-checks", type=int, default=12000,
+                     help="abduction mode: cap on certification checks per example (default 12000)")
+    fol.add_argument("--lean", action="store_true", help="reference mode: certify verdicts with Lean")
+    fol.add_argument("--base-url")
+    fol.add_argument("--model")
+    fol.add_argument("--api-key-env", default="LLM_API_KEY", help="name of the variable holding the key")
+    fol.add_argument("--limit", type=int, help="model and abduction modes: only the first N examples")
+    fol.add_argument("--no-json-mode", action="store_true",
+                     help="do not send response_format=json_object (for providers that reject it)")
+    fol.add_argument("--temperature", type=float, default=0.0, help="model modes: sampling temperature (default 0)")
+    fol.add_argument("--no-temperature", action="store_true",
+                     help="send no temperature (for models that accept only their default)")
+    fol.add_argument("--max-tokens", type=int, default=4000, help="model modes: reply length limit (default 4000)")
+    fol.set_defaults(run=_folio)
+
+    for name, text in (("compare", "paired comparison of two model runs (McNemar), as JSON"),
+                       ("report", "the same comparison as a Markdown summary")):
+        compare = commands.add_parser(name, help=text)
+        compare.add_argument("first", help="output directory of one run, e.g. results/direct")
+        compare.add_argument("second", help="output directory of the other, e.g. results/llm")
+        compare.set_defaults(run=_compare)
+
+    args = parser.parse_args(argv)
+    return args.run(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())