knowledge-worker 0.6.0__py3-none-any.whl

mygraph/owl_io.py

@@ -0,0 +1,202 @@
+"""
+owl_io.py — Turtle (OWL) sibling serialization for the graph.
+
+JSON stays canonical. mygraph.ttl is generated from JSON at any time and can be
+re-imported losslessly (round-trip on node + edge counts is a hard test).
+
+Mapping follows the graph model in SPEC.md and the pipeline in DESIGN.md:
+
+  - Node type           → owl:Class under rb:Concept
+  - Edge type           → owl:ObjectProperty
+  - Node id             → IRI <http://mygraph.local/{id}>
+  - Node label / body   → rdfs:label / rdfs:comment
+  - Edge metadata       → rb:Assertion (reified) with rb:confidence,
+                          rb:excerpt, rb:sourceId, rb:createdAt, rb:lastSeen
+  - Source node         → rb:Source (subclass of dcterms:ProvenanceEntity)
+
+Requires `rdflib` (`pip install -e ".[rdf]"`).
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from mygraph import Graph, Node, Edge, NODE_TYPES, EDGE_TYPES, resolve_graph_path
+
+NS = "http://mygraph.local/"
+RB = "http://mygraph.local/schema#"
+DCTERMS = "http://purl.org/dc/terms/"
+
+
+def _require_rdflib():
+    try:
+        import rdflib  # type: ignore
+        return rdflib
+    except ImportError as e:
+        raise SystemExit(
+            "owl_io: `rdflib` is not installed. Run:\n"
+            '    pip install -e ".[rdf]"'
+        ) from e
+
+
+def _iri(rdflib, suffix: str):
+    # rdflib URIRef accepts strings; we keep IDs verbatim (slug already URL-safe except `:`)
+    safe = suffix.replace(":", "/")
+    return rdflib.URIRef(NS + safe)
+
+
+def to_turtle(g: Graph) -> str:
+    rdflib = _require_rdflib()
+    from rdflib import Graph as RG, Literal, RDF, RDFS, OWL, Namespace
+    rg = RG()
+    rb_ns = Namespace(RB)
+    dc_ns = Namespace(DCTERMS)
+    rg.bind("rb", rb_ns)
+    rg.bind("dcterms", dc_ns)
+    rg.bind("mg", Namespace(NS))
+
+    # ontology: classes/properties include public schema plus any legacy/private
+    # types already present in the graph.
+    rg.add((rb_ns.Concept, RDF.type, OWL.Class))
+    rg.add((rb_ns.Source, RDF.type, OWL.Class))
+    rg.add((rb_ns.Source, RDFS.subClassOf, dc_ns.ProvenanceEntity))
+    node_types = NODE_TYPES | {n.type for n in g.nodes.values()}
+    edge_types = EDGE_TYPES | {e.type for e in g.edges}
+    for t in sorted(node_types):
+        cls = rb_ns[t.capitalize()]
+        rg.add((cls, RDF.type, OWL.Class))
+        if t == "source":
+            rg.add((cls, RDFS.subClassOf, rb_ns.Source))
+        else:
+            rg.add((cls, RDFS.subClassOf, rb_ns.Concept))
+    for t in sorted(edge_types):
+        rg.add((rb_ns[t], RDF.type, OWL.ObjectProperty))
+
+    # nodes
+    for nid, n in g.nodes.items():
+        iri = _iri(rdflib, nid)
+        rg.add((iri, RDF.type, rb_ns[n.type.capitalize()]))
+        rg.add((iri, RDFS.label, Literal(n.label)))
+        if n.body:
+            rg.add((iri, RDFS.comment, Literal(n.body)))
+        rg.add((iri, rb_ns.confidence, Literal(n.confidence)))
+        rg.add((iri, rb_ns.createdAt, Literal(n.created_at)))
+        rg.add((iri, rb_ns.nodeId, Literal(nid)))
+
+    # edges (direct triple + reified rb:Assertion holding metadata)
+    for i, e in enumerate(g.edges):
+        s = _iri(rdflib, e.src)
+        o = _iri(rdflib, e.dst)
+        p = rb_ns[e.type]
+        rg.add((s, p, o))
+        # reify
+        a_iri = rdflib.URIRef(f"{NS}_assertion/{i}")
+        rg.add((a_iri, RDF.type, rb_ns.Assertion))
+        rg.add((a_iri, RDF.subject, s))
+        rg.add((a_iri, RDF.predicate, p))
+        rg.add((a_iri, RDF.object, o))
+        rg.add((a_iri, rb_ns.sourceId, Literal(e.source_id)))
+        rg.add((a_iri, rb_ns.confidence, Literal(e.confidence)))
+        rg.add((a_iri, rb_ns.createdAt, Literal(e.created_at)))
+        rg.add((a_iri, rb_ns.lastSeen, Literal(e.last_seen)))
+        rg.add((a_iri, rb_ns.edgeType, Literal(e.type)))
+        rg.add((a_iri, rb_ns.srcId, Literal(e.src)))
+        rg.add((a_iri, rb_ns.dstId, Literal(e.dst)))
+        if e.excerpt:
+            rg.add((a_iri, rb_ns.excerpt, Literal(e.excerpt)))
+
+    return rg.serialize(format="turtle")
+
+
+def from_turtle(path: Path) -> Graph:
+    """Reimport a graph from Turtle. Reads from the rb:Assertion reifications
+    (which carry the full edge metadata) — direct triples are redundant."""
+    rdflib = _require_rdflib()
+    from rdflib import Graph as RG, Literal, RDF, RDFS, Namespace
+    rg = RG()
+    rg.parse(str(path), format="turtle")
+    rb_ns = Namespace(RB)
+
+    nodes: dict[str, Node] = {}
+    # iterate all subjects with rb:nodeId set
+    for s, _, lit in rg.triples((None, rb_ns.nodeId, None)):
+        nid = str(lit)
+        # type from rdf:type (capitalize → lowercase mapping)
+        t_iri = next(rg.objects(s, RDF.type), None)
+        type_str = (str(t_iri).rsplit("#", 1)[-1] if t_iri else "idea").lower()
+        label = str(next(rg.objects(s, RDFS.label), Literal("")))
+        body = str(next(rg.objects(s, RDFS.comment), Literal("")))
+        conf = str(next(rg.objects(s, rb_ns.confidence), Literal("medium")))
+        created = str(next(rg.objects(s, rb_ns.createdAt),
+                           Literal("1970-01-01T00:00:00+00:00")))
+        nodes[nid] = Node(id=nid, type=type_str, label=label, body=body,
+                          confidence=conf, created_at=created)
+
+    edges: list[Edge] = []
+    for a_iri, _, _ in rg.triples((None, RDF.type, rb_ns.Assertion)):
+        src = str(next(rg.objects(a_iri, rb_ns.srcId), Literal("")))
+        dst = str(next(rg.objects(a_iri, rb_ns.dstId), Literal("")))
+        etype = str(next(rg.objects(a_iri, rb_ns.edgeType), Literal("")))
+        if not etype:
+            continue
+        sid = str(next(rg.objects(a_iri, rb_ns.sourceId), Literal("")))
+        conf = str(next(rg.objects(a_iri, rb_ns.confidence), Literal("medium")))
+        created = str(next(rg.objects(a_iri, rb_ns.createdAt),
+                           Literal("1970-01-01T00:00:00+00:00")))
+        last = str(next(rg.objects(a_iri, rb_ns.lastSeen), Literal(created)))
+        excerpt = str(next(rg.objects(a_iri, rb_ns.excerpt), Literal("")))
+        edges.append(Edge(src=src, dst=dst, type=etype, source_id=sid,
+                          excerpt=excerpt, confidence=conf,
+                          created_at=created, last_seen=last))
+    return Graph(nodes=nodes, edges=edges)
+
+
+def round_trip_test(graph_path: Path | None = None) -> tuple[bool, str]:
+    import tempfile
+    active_path = graph_path or Path(resolve_graph_path())
+    g = Graph.load(str(active_path))
+    ttl = to_turtle(g)
+    with tempfile.NamedTemporaryFile("w", suffix=".ttl", delete=False) as f:
+        f.write(ttl)
+        tmp = Path(f.name)
+    try:
+        g2 = from_turtle(tmp)
+    finally:
+        try:
+            tmp.unlink()
+        except OSError:
+            pass
+    n_match = len(g.nodes) == len(g2.nodes)
+    e_match = len(g.edges) == len(g2.edges)
+    if n_match and e_match:
+        return True, f"OK: {len(g.nodes)} nodes / {len(g.edges)} edges round-tripped"
+    return False, (f"MISMATCH: orig {len(g.nodes)}/{len(g.edges)} "
+                   f"vs reimport {len(g2.nodes)}/{len(g2.edges)}")
+
+
+def run_export(args: list[str]) -> int:
+    if "--ttl" not in args:
+        print("Usage: mykg export --ttl [--graph <path>] [--out <path>] [--round-trip]")
+        return 1
+    graph_path = Path(resolve_graph_path())
+    if "--graph" in args:
+        i = args.index("--graph")
+        graph_path = Path(args[i + 1]).expanduser().resolve()
+    out = graph_path.with_suffix(".ttl")
+    if "--out" in args:
+        i = args.index("--out")
+        out = Path(args[i + 1]).expanduser().resolve()
+    g = Graph.load(str(graph_path))
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(to_turtle(g), encoding="utf-8")
+    print(f"export: wrote {out}")
+    if "--round-trip" in args:
+        ok, msg = round_trip_test(graph_path)
+        print(f"round-trip: {msg}")
+        return 0 if ok else 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run_export(sys.argv[1:]))

mygraph/review.py

@@ -0,0 +1,151 @@
+"""
+review.py — Stage 3 of the v1 ingest pipeline.
+
+Interactive terminal loop over validated candidates. Keys:
+  [a]ccept   merge into the graph
+  [r]eject   skip; record the rejection
+  [e]dit     pop $EDITOR on the candidate JSON, then re-validate this candidate
+  [s]kip     defer to next session
+  [q]uit     stop reviewing (already-merged stay merged)
+
+Idempotent: re-running on the same source_id skips already-merged node IDs.
+
+Non-interactive modes (for headless testing / dispatch):
+  --auto-accept-high   accept everything with confidence == "high"
+  --auto-accept-all    accept every accepted candidate
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Iterable
+
+from mygraph import Graph
+try:
+    from .validator import validate
+    from .eval_log import append as eval_append
+except ImportError:  # direct script execution
+    from validator import validate
+    from eval_log import append as eval_append
+
+
+def _print_node(node: dict) -> None:
+    print(f"\n[{node['type']}] {node['id']}")
+    print(f"  label     : {node['label']}")
+    if node.get("body"):
+        print(f"  body      : {node['body']}")
+    print(f"  confidence: {node.get('confidence')}")
+    if node.get("excerpt"):
+        print(f"  excerpt   : \"{node['excerpt']}\"")
+
+
+def _print_edge(edge: dict) -> None:
+    print(f"\n[edge] {edge['src']} --{edge['type']}--> {edge['dst']}")
+    print(f"  confidence: {edge.get('confidence')}")
+    if edge.get("excerpt"):
+        print(f"  excerpt   : \"{edge['excerpt']}\"")
+
+
+def _edit_in_editor(payload: dict) -> dict:
+    editor = os.environ.get("EDITOR", "vi")
+    with tempfile.NamedTemporaryFile("w+", suffix=".json", delete=False, encoding="utf-8") as f:
+        json.dump(payload, f, indent=2)
+        path = f.name
+    try:
+        subprocess.run([editor, path], check=False)
+        with open(path, encoding="utf-8") as f:
+            return json.load(f)
+    finally:
+        os.unlink(path)
+
+
+def _already_merged_ids(g: Graph, source_id: str) -> set[str]:
+    """Nodes that already have a MENTIONED_IN edge to this source."""
+    out = set()
+    for e in g.edges:
+        if e.type == "MENTIONED_IN" and e.dst == source_id:
+            out.add(e.src)
+        elif e.type == "MENTIONED_IN" and e.src == source_id:
+            out.add(e.dst)
+    return out
+
+
+def _ask(prompt: str, valid: set[str]) -> str:
+    while True:
+        try:
+            ans = input(prompt).strip().lower()
+        except EOFError:
+            return "q"
+        if ans in valid:
+            return ans
+        print(f"  ? choose one of: {sorted(valid)}")
+
+
+def review(validated: dict, source_text: str,
+           auto_accept_high: bool = False,
+           auto_accept_all: bool = False) -> dict:
+    """
+    Returns the user-approved subset of `validated` (same shape).
+    Edges whose endpoints aren't approved get filtered after node decisions.
+    """
+    g = Graph.load()
+    src = validated["source"]
+    already = _already_merged_ids(g, src["id"])
+
+    approved_nodes: list[dict] = []
+    decisions: list[dict] = []  # for eval_log
+
+    auto = auto_accept_all or auto_accept_high
+    for node in validated.get("nodes", []):
+        if node["id"] in already:
+            decisions.append({"kind": "review", "verdict": "skip_already_merged",
+                              "candidate_id": node["id"], "source_id": src["id"],
+                              "extractor_confidence": node.get("confidence")})
+            continue
+        if auto:
+            verdict = "accept" if (auto_accept_all or node.get("confidence") == "high") else "skip"
+            user_edit = None
+        else:
+            _print_node(node)
+            choice = _ask("  [a]ccept [r]eject [e]dit [s]kip [q]uit > ",
+                          {"a", "r", "e", "s", "q"})
+            user_edit = None
+            if choice == "q":
+                break
+            if choice == "e":
+                edited = _edit_in_editor(node)
+                # re-validate just this candidate against the source
+                subset = {"source": src, "nodes": [edited], "edges": []}
+                v_payload, _ = validate(subset, source_text)
+                if v_payload["nodes"]:
+                    node = v_payload["nodes"][0]
+                    user_edit = edited
+                    _print_node(node)
+                    choice = _ask("  After edit: [a]ccept [r]eject [s]kip > ", {"a", "r", "s"})
+                else:
+                    print("  (edit failed validation, skipping)")
+                    choice = "s"
+            verdict = {"a": "accept", "r": "reject", "s": "skip"}.get(choice, "skip")
+        decisions.append({"kind": "review", "verdict": verdict,
+                          "candidate_id": node["id"], "source_id": src["id"],
+                          "extractor_confidence": node.get("confidence"),
+                          "user_edit": user_edit})
+        if verdict == "accept":
+            approved_nodes.append(node)
+
+    approved_node_ids = {n["id"] for n in approved_nodes} | set(g.nodes.keys()) | {src["id"]}
+    approved_edges: list[dict] = []
+    for edge in validated.get("edges", []):
+        if edge["src"] in approved_node_ids and edge["dst"] in approved_node_ids:
+            approved_edges.append(edge)
+
+    for d in decisions:
+        eval_append(d)
+
+    return {"source": src, "nodes": approved_nodes, "edges": approved_edges,
+            "_meta": validated.get("_meta", {})}

mygraph/validator.py

@@ -0,0 +1,149 @@
+"""
+validator.py — Stage 2 of the v1 ingest pipeline.
+
+Takes the extractor's candidates.json and runs deterministic checks:
+  - shape (per the extractor's tool_schema)
+  - excerpt verification (`high` confidence MUST substring-match the source markdown,
+    after whitespace normalization)
+  - orphan edge check (src/dst must resolve to existing node OR another candidate)
+  - ID format (`^[a-z]+:[a-z0-9-]+$`)
+
+Output: a validated payload (same shape as input) plus a manifest of
+  accepted / demoted-with-reason / rejected-with-reason.
+
+No external deps. Lightweight schema check (we control both producer and consumer).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from mygraph import Graph, NODE_TYPES, EDGE_TYPES
+
+ID_RE = re.compile(r"^[a-z]+:[a-z0-9-]+$")
+WS_RE = re.compile(r"\s+")
+
+
+def _norm(s: str) -> str:
+    return WS_RE.sub(" ", s).strip().lower()
+
+
+@dataclass
+class Manifest:
+    accepted_nodes: list[dict] = field(default_factory=list)
+    accepted_edges: list[dict] = field(default_factory=list)
+    demoted_nodes: list[tuple[dict, str]] = field(default_factory=list)
+    rejected_nodes: list[tuple[dict, str]] = field(default_factory=list)
+    rejected_edges: list[tuple[dict, str]] = field(default_factory=list)
+
+    def summary(self) -> str:
+        return (
+            f"  accepted : {len(self.accepted_nodes)} nodes / {len(self.accepted_edges)} edges\n"
+            f"  demoted  : {len(self.demoted_nodes)} nodes\n"
+            f"  rejected : {len(self.rejected_nodes)} nodes / {len(self.rejected_edges)} edges"
+        )
+
+
+def _check_shape(payload: dict) -> list[str]:
+    errs = []
+    if not isinstance(payload, dict):
+        return ["payload is not a dict"]
+    if "source" not in payload or not isinstance(payload["source"], dict):
+        errs.append("missing source object")
+    else:
+        for k in ("id", "label", "body"):
+            if k not in payload["source"]:
+                errs.append(f"source missing field: {k}")
+    for key in ("nodes", "edges"):
+        if key not in payload or not isinstance(payload[key], list):
+            errs.append(f"missing {key} list")
+    return errs
+
+
+def validate(payload: dict, source_text: str) -> tuple[dict, Manifest]:
+    """Return (validated_payload, manifest). validated_payload mutates confidences and drops rejects."""
+    shape_errs = _check_shape(payload)
+    if shape_errs:
+        raise ValueError("validator: malformed payload → " + "; ".join(shape_errs))
+
+    g = Graph.load()
+    manifest = Manifest()
+    src_norm = _norm(source_text)
+
+    # validate Source
+    src = payload["source"]
+    if not ID_RE.match(src["id"]) or not src["id"].startswith("source:"):
+        raise ValueError(f"validator: invalid source id: {src['id']!r}")
+
+    # validate nodes
+    valid_nodes: list[dict] = []
+    candidate_ids: set[str] = {src["id"]}
+    for node in payload.get("nodes", []):
+        nid = node.get("id", "")
+        if not ID_RE.match(nid):
+            manifest.rejected_nodes.append((node, "id_format"))
+            continue
+        if node.get("type") not in NODE_TYPES:
+            manifest.rejected_nodes.append((node, f"bad_type:{node.get('type')}"))
+            continue
+        if node.get("confidence") not in {"high", "medium", "low"}:
+            manifest.rejected_nodes.append((node, "bad_confidence"))
+            continue
+        # provenance-or-bust: high → must have excerpt + must substring-match source
+        excerpt = (node.get("excerpt") or "").strip()
+        if node["confidence"] == "high":
+            if not excerpt:
+                node["confidence"] = "low"
+                manifest.demoted_nodes.append((node, "no_excerpt"))
+            elif _norm(excerpt) not in src_norm:
+                node["confidence"] = "low"
+                manifest.demoted_nodes.append((node, "excerpt_not_in_source"))
+        candidate_ids.add(nid)
+        valid_nodes.append(node)
+        manifest.accepted_nodes.append(node)
+
+    # validate edges
+    valid_edges: list[dict] = []
+    for edge in payload.get("edges", []):
+        if edge.get("type") not in EDGE_TYPES:
+            manifest.rejected_edges.append((edge, f"bad_type:{edge.get('type')}"))
+            continue
+        if edge.get("confidence") not in {"high", "medium", "low"}:
+            manifest.rejected_edges.append((edge, "bad_confidence"))
+            continue
+        for endpoint_key in ("src", "dst"):
+            ep = edge.get(endpoint_key, "")
+            if not ID_RE.match(ep):
+                manifest.rejected_edges.append((edge, f"{endpoint_key}_id_format"))
+                break
+            if ep not in g.nodes and ep not in candidate_ids:
+                manifest.rejected_edges.append((edge, f"orphan_{endpoint_key}:{ep}"))
+                break
+        else:
+            valid_edges.append(edge)
+            manifest.accepted_edges.append(edge)
+
+    validated = dict(payload)
+    validated["nodes"] = valid_nodes
+    validated["edges"] = valid_edges
+    return validated, manifest
+
+
+def main():
+    import json
+    import sys
+    if len(sys.argv) < 3:
+        print("Usage: python validator.py <candidates.json> <source.md>")
+        return 1
+    payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
+    src_text = Path(sys.argv[2]).read_text(encoding="utf-8")
+    _, manifest = validate(payload, src_text)
+    print(manifest.summary())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())