knowledge-worker 0.6.0__py3-none-any.whl

mygraph/audit.py

@@ -0,0 +1,137 @@
+"""
+audit.py — structural verifier for external-query audits (e.g. copilot_response_audit.md).
+
+No LLM. Just structural checks against the graph:
+  - confidence labels surfaced for non-high nodes?
+  - provenance (source:*) cited?
+  - completeness: did the response list all expected nodes for a typed question?
+
+Outputs JSONL eval_record entries appended to eval_record.jsonl.
+
+Usage:
+    python audit.py copilot_response_audit.md
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mygraph import Graph
+
+HERE = Path(__file__).parent
+EVAL_LOG = HERE / "eval_record.jsonl"
+
+
+def parse_audit(md_text: str) -> list[dict]:
+    blocks = re.split(r'\n## \d+\.', md_text)[1:]
+    out = []
+    for b in blocks:
+        q = re.search(r'\*\*User:\*\*\s*(.+?)(?=\*\*Copilot:\*\*)', b, re.DOTALL)
+        r = re.search(r'\*\*Copilot:\*\*\s*(.+?)(?=\*\*Evaluation:\*\*|\Z)', b, re.DOTALL)
+        e = re.search(r'\*\*Evaluation:\*\*\s*(.+?)\Z', b, re.DOTALL)
+        out.append({
+            "question": q.group(1).strip() if q else "",
+            "response": r.group(1).strip() if r else "",
+            "self_eval": e.group(1).strip() if e else "",
+        })
+    return out
+
+
+def referenced_nodes(text: str, g: Graph) -> list[str]:
+    ids = set(re.findall(r'\b([a-z]+:[a-z0-9-]+)\b', text))
+    low = text.lower()
+    for nid, n in g.nodes.items():
+        if len(n.label) > 4 and n.label.lower() in low:
+            ids.add(nid)
+    return sorted(i for i in ids if i in g.nodes)
+
+
+def check_confidence(refs, g, resp):
+    issues = []
+    low = resp.lower()
+    for r in refs:
+        n = g.nodes[r]
+        if n.confidence != "high" and n.confidence not in low and "confidence" not in low:
+            issues.append(f"unflagged_{n.confidence}_confidence:{r}")
+    return issues
+
+
+def check_provenance(refs, resp):
+    if not refs:
+        return []
+    return [] if "source:" in resp.lower() else ["no_source_cited"]
+
+
+def check_completeness(question, refs, g, resp):
+    q = question.lower()
+    issues = []
+    typed_checks = [
+        ("goal", "goal"),
+        ("decid", "decision"),
+        ("idea", "idea"),
+        ("question", "question"),
+    ]
+    for keyword, node_type in typed_checks:
+        if keyword in q and "?" in q:
+            all_of = [nid for nid, n in g.nodes.items() if n.type == node_type]
+            if not all_of:
+                continue
+            listed = [r for r in refs if r in all_of]
+            if len(listed) < len(all_of):
+                issues.append(f"incomplete_{node_type}_listing:{len(listed)}/{len(all_of)}")
+    return issues
+
+
+def audit(audit_path: Path) -> list[dict]:
+    g = Graph.load()
+    md = audit_path.read_text(encoding="utf-8")
+    blocks = parse_audit(md)
+    records = []
+    ts = datetime.now(timezone.utc).isoformat()
+    for i, b in enumerate(blocks, 1):
+        refs = referenced_nodes(b["response"], g)
+        misses = (
+            check_confidence(refs, g, b["response"])
+            + check_provenance(refs, b["response"])
+            + check_completeness(b["question"], refs, g, b["response"])
+        )
+        records.append({
+            "ts": ts,
+            "kind": "external_query",
+            "audit_source": audit_path.name,
+            "q_index": i,
+            "question": b["question"][:200],
+            "response_excerpt": b["response"][:300],
+            "self_eval": b["self_eval"][:200],
+            "referenced_nodes": refs,
+            "claude_verdict": "ok" if not misses else "miss",
+            "misses": misses,
+        })
+    with EVAL_LOG.open("a", encoding="utf-8") as f:
+        for r in records:
+            f.write(json.dumps(r) + "\n")
+    return records
+
+
+def main(argv):
+    path = Path(argv[1]) if len(argv) > 1 else HERE / "copilot_response_audit.md"
+    if not path.exists():
+        print(f"Not found: {path}")
+        return 1
+    records = audit(path)
+    misses = [r for r in records if r["misses"]]
+    print(f"Wrote {len(records)} eval_records -> {EVAL_LOG}")
+    print(f"  {len(misses)}/{len(records)} responses flagged with misses\n")
+    for r in misses:
+        print(f"  Q{r['q_index']}: {r['question'][:60]}")
+        for m in r["misses"]:
+            print(f"      - {m}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

mygraph/check.py

@@ -0,0 +1,273 @@
+"""
+check.py — v1 M2 offline health checks.
+
+Subcommands (all write JSONL records to eval_record.jsonl):
+
+  --provenance              hard invariant. Any node (except `source`) without a
+                            MENTIONED_IN edge → kind: provenance_violation. Any
+                            edge without source_id → same.
+  --stale-edges [--days N]  edges with last_seen older than N days (default 90)
+                            → kind: stale_candidate.
+  --pairs N                 pick N random non-adjacent node pairs, ask the LLM
+                            "is X related to Y? if yes, by what predicate?".
+                            Logs kind: relational_probe.
+  --source-candidates DIR   read recent .md/.txt files in DIR; ask the LLM if
+                            any look like Sources we should ingest. Logs
+                            kind: source_candidate. Never auto-ingests.
+
+Default (no subcommand): runs --provenance and --stale-edges. LLM-bound checks
+use the configured Anthropic provider; they're skipped if no supported provider
+env is present.
+"""
+
+from __future__ import annotations
+
+import random
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from mygraph import Graph
+try:
+    from .anthropic_client import anthropic_configured, get_anthropic_client
+    from .eval_log import append as eval_append, append_many
+except ImportError:  # direct script execution
+    from anthropic_client import anthropic_configured, get_anthropic_client
+    from eval_log import append as eval_append, append_many
+
+
+# ------------ provenance ------------------------------------------------------
+
+def check_provenance(g: Graph) -> list[dict]:
+    """Return list of violation records (also appended to eval_record.jsonl)."""
+    violations = []
+    mentioned_node_ids = {e.src for e in g.edges if e.type == "MENTIONED_IN"}
+    mentioned_node_ids |= {e.dst for e in g.edges if e.type == "MENTIONED_IN"}
+    for nid, n in g.nodes.items():
+        if n.type == "source":
+            continue
+        if nid not in mentioned_node_ids:
+            violations.append({
+                "kind": "provenance_violation",
+                "subkind": "node_without_source",
+                "node_id": nid,
+                "node_type": n.type,
+                "label": n.label,
+            })
+    for i, e in enumerate(g.edges):
+        if not e.source_id:
+            violations.append({
+                "kind": "provenance_violation",
+                "subkind": "edge_without_source_id",
+                "edge_index": i,
+                "src": e.src, "dst": e.dst, "type": e.type,
+            })
+    append_many(violations)
+    return violations
+
+
+# ------------ stale edges -----------------------------------------------------
+
+def check_stale_edges(g: Graph, days: int = 90) -> list[dict]:
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    stale = []
+    for i, e in enumerate(g.edges):
+        try:
+            ls = datetime.fromisoformat(e.last_seen)
+        except (ValueError, TypeError):
+            continue
+        if ls < cutoff:
+            stale.append({
+                "kind": "stale_candidate",
+                "edge_index": i,
+                "src": e.src, "dst": e.dst, "type": e.type,
+                "last_seen": e.last_seen,
+                "age_days": (datetime.now(timezone.utc) - ls).days,
+            })
+    append_many(stale)
+    return stale
+
+
+# ------------ relational probe (LLM) -----------------------------------------
+
+PAIR_PROMPT = """\
+You are evaluating a personal knowledge graph. Two nodes are below. Decide:
+  (a) Are these conceptually related?
+  (b) If yes, what predicate name from this set best fits?
+       {edge_types}
+
+Respond as a single JSON object with keys:
+  related   : true | false
+  predicate : one of the predicates above, or null if related=false
+  rationale : one short sentence
+  confidence: high | medium | low
+
+NODE A: id={a_id}  type={a_type}  label={a_label}
+        body: {a_body}
+NODE B: id={b_id}  type={b_type}  label={b_label}
+        body: {b_body}
+"""
+
+
+def _call_claude_json(prompt: str) -> dict | None:
+    """Lightweight Claude call returning parsed JSON; None if no LLM config."""
+    if not anthropic_configured():
+        return None
+    try:
+        client, config = get_anthropic_client()
+    except RuntimeError as e:
+        print(f"check: {e}; skipping LLM-bound checks.")
+        return None
+    import json as _json
+    resp = client.messages.create(
+        model=config.model, max_tokens=400,
+        messages=[{"role": "user", "content": prompt + "\n\nReturn ONLY JSON."}],
+    )
+    text = "".join(getattr(b, "text", "") for b in resp.content)
+    text = text.strip()
+    # be forgiving: strip ```json fences
+    if text.startswith("```"):
+        text = text.strip("`")
+        text = text.split("\n", 1)[1] if "\n" in text else text
+        text = text.rsplit("```", 1)[0] if text.endswith("```") else text
+    try:
+        return _json.loads(text)
+    except _json.JSONDecodeError:
+        return {"_raw": text, "_parse_error": True}
+
+
+def check_pairs(g: Graph, k: int = 10) -> list[dict]:
+    from mygraph import EDGE_TYPES
+    if not anthropic_configured():
+        print("check --pairs: no Anthropic provider env configured; skipping.")
+        return []
+    adj = set()
+    for e in g.edges:
+        adj.add((e.src, e.dst))
+        adj.add((e.dst, e.src))
+    ids = [nid for nid, n in g.nodes.items() if n.type != "source"]
+    pairs = []
+    attempts = 0
+    while len(pairs) < k and attempts < k * 20:
+        attempts += 1
+        a, b = random.sample(ids, 2)
+        if (a, b) in adj:
+            continue
+        if (a, b) in pairs or (b, a) in pairs:
+            continue
+        pairs.append((a, b))
+    records = []
+    for a, b in pairs:
+        na, nb = g.nodes[a], g.nodes[b]
+        prompt = PAIR_PROMPT.format(
+            edge_types=", ".join(sorted(EDGE_TYPES)),
+            a_id=a, a_type=na.type, a_label=na.label, a_body=na.body[:200],
+            b_id=b, b_type=nb.type, b_label=nb.label, b_body=nb.body[:200],
+        )
+        result = _call_claude_json(prompt)
+        records.append({
+            "kind": "relational_probe",
+            "a_id": a, "b_id": b,
+            "claude_result": result,
+        })
+    append_many(records)
+    return records
+
+
+# ------------ source candidacy (LLM) -----------------------------------------
+
+SOURCE_CANDIDATE_PROMPT = """\
+A markdown/text document is below. Decide whether it should be ingested as a
+Source into the user's personal knowledge graph. Respond as JSON:
+
+  ingest_recommendation : "yes" | "no" | "maybe"
+  rationale             : one short sentence
+  candidate_concepts    : list of 1-5 plain-English concept labels you'd extract
+                          if ingested (empty list if no)
+
+DOCUMENT (filename: {fname}):
+---
+{content}
+---
+"""
+
+
+def check_source_candidates(g: Graph, dir_path: Path) -> list[dict]:
+    if not anthropic_configured():
+        print("check --source-candidates: no Anthropic provider env configured; skipping.")
+        return []
+    if not dir_path.is_dir():
+        print(f"check --source-candidates: not a directory: {dir_path}")
+        return []
+    existing_source_paths = {n.body for n in g.nodes.values() if n.type == "source"}
+    records = []
+    for p in sorted(dir_path.glob("*.md")) + sorted(dir_path.glob("*.txt")):
+        if str(p) in existing_source_paths:
+            continue
+        content = p.read_text(encoding="utf-8")[:8000]
+        prompt = SOURCE_CANDIDATE_PROMPT.format(fname=p.name, content=content)
+        result = _call_claude_json(prompt)
+        records.append({
+            "kind": "source_candidate",
+            "path": str(p),
+            "claude_result": result,
+        })
+    append_many(records)
+    return records
+
+
+# ------------ CLI dispatch ----------------------------------------------------
+
+def run_check(args: list[str]) -> int:
+    g = Graph.load()
+    flags = list(args)
+
+    # parse value-bearing flags
+    days = 90
+    if "--days" in flags:
+        i = flags.index("--days")
+        days = int(flags[i + 1]); del flags[i:i + 2]
+    pairs_n = 0
+    if "--pairs" in flags:
+        i = flags.index("--pairs")
+        try:
+            pairs_n = int(flags[i + 1]); del flags[i:i + 2]
+        except (ValueError, IndexError):
+            pairs_n = 10; del flags[i:i + 1]
+    source_dir = None
+    if "--source-candidates" in flags:
+        i = flags.index("--source-candidates")
+        if i + 1 < len(flags):
+            source_dir = Path(flags[i + 1]).expanduser().resolve()
+            del flags[i:i + 2]
+        else:
+            print("check: --source-candidates needs a directory")
+            return 1
+
+    only = set(f for f in flags if f.startswith("--"))
+    run_all = not only and not pairs_n and not source_dir
+
+    rc = 0
+    if run_all or "--provenance" in only:
+        v = check_provenance(g)
+        print(f"provenance violations: {len(v)}")
+        for r in v[:10]:
+            print(f"  - {r['subkind']}: {r.get('node_id') or r.get('src')+'->'+r.get('dst')}")
+        if v:
+            rc = 2  # non-zero exit on hard-invariant break
+    if run_all or "--stale-edges" in only:
+        s = check_stale_edges(g, days=days)
+        print(f"stale edges (>{days}d): {len(s)}")
+        for r in s[:10]:
+            print(f"  - {r['src']} --{r['type']}--> {r['dst']}  ({r['age_days']}d)")
+    if pairs_n:
+        p = check_pairs(g, k=pairs_n)
+        print(f"relational probes: {len(p)} run")
+    if source_dir:
+        sc = check_source_candidates(g, source_dir)
+        print(f"source candidates: {len(sc)} evaluated")
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(run_check(sys.argv[1:]))