knowledge-worker 0.6.0__py3-none-any.whl

mygraph/export_context.py

@@ -0,0 +1,124 @@
+"""
+export_context.py — generate a compact LLM-ready context snapshot of mygraph.
+Usage: mykg context [--out context.md] [--max-ideas 20]
+"""
+import argparse
+import datetime
+import json
+import sys
+from collections import defaultdict
+
+try:
+    from .mygraph import resolve_graph_path
+except ImportError:  # direct script execution: python mygraph/export_context.py
+    from mygraph import resolve_graph_path
+
+
+def load(path=None):
+    path = resolve_graph_path(path)
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+def export_context(g, max_ideas=20):
+    nodes = g["nodes"]
+    edges = g["edges"]
+
+    # Build incoming edge count per node
+    in_edges = defaultdict(list)
+    for e in edges:
+        in_edges[e["dst"]].append(e)
+
+    def by_type(t):
+        return [n for n in nodes.values() if n.get("type") == t]
+
+    def conf_marker(c):
+        if c == "low": return " WARN"
+        if c == "medium": return " ~"
+        return ""
+
+    lines = []
+    lines.append("# mygraph - Context Snapshot")
+    lines.append(f"*Generated: {datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')} | "
+                 f"{len(nodes)} nodes, {len(edges)} edges*\n")
+
+    # Goals
+    goals = by_type("goal")
+    if goals:
+        lines.append("## Goals")
+        for n in goals:
+            lines.append(f"- **{n['label']}**{conf_marker(n.get('confidence',''))}")
+            if n.get("body"):
+                lines.append(f"  {n['body'][:120]}")
+        lines.append("")
+
+    # Decisions (high confidence only to keep it tight)
+    decisions = [n for n in by_type("decision") if n.get("confidence") != "low"]
+    if decisions:
+        lines.append("## Key Decisions")
+        for n in decisions[:20]:
+            lines.append(f"- **{n['label']}**")
+            if n.get("body"):
+                lines.append(f"  {n['body'][:100]}")
+        lines.append("")
+
+    # Ideas: sort by incoming edge count (most connected first)
+    ideas = by_type("idea")
+    ideas_sorted = sorted(ideas, key=lambda n: -len(in_edges.get(n["id"], [])))
+    lines.append("## Ideas")
+    for n in ideas_sorted[:max_ideas]:
+        edge_count = len(in_edges.get(n["id"], []))
+        marker = conf_marker(n.get("confidence",""))
+        lines.append(f"- **{n['label']}**{marker} *(connections: {edge_count})*")
+        if n.get("body"):
+            lines.append(f"  {n['body'][:120]}")
+    lines.append("")
+
+    # Topics: k-core proxy, more than 1 incoming edge.
+    topics = [n for n in by_type("topic") if len(in_edges.get(n["id"],[])) > 1]
+    topics_sorted = sorted(topics, key=lambda n: -len(in_edges.get(n["id"],[])))
+    if topics_sorted:
+        lines.append("## Core Topics")
+        for n in topics_sorted[:20]:
+            lines.append(f"- {n['label']} *(x{len(in_edges[n['id']])})*")
+        lines.append("")
+
+    # Recent sources (last 5)
+    sources = sorted(by_type("source"), key=lambda n: n.get("created_at",""), reverse=True)[:5]
+    if sources:
+        lines.append("## Recent Sources")
+        for n in sources:
+            lines.append(f"- {n['label']}")
+        lines.append("")
+
+    # Questions
+    questions = by_type("question")
+    if questions:
+        lines.append("## Open Questions")
+        for n in questions[:10]:
+            lines.append(f"- {n['label']}{conf_marker(n.get('confidence',''))}")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def run_export_context(args: list[str]) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--graph", default=None)
+    parser.add_argument("--out", default=None)
+    parser.add_argument("--max-ideas", type=int, default=20)
+    parsed = parser.parse_args(args)
+
+    g = load(parsed.graph)
+    text = export_context(g, max_ideas=parsed.max_ideas)
+
+    if parsed.out:
+        with open(parsed.out, "w", encoding="utf-8") as f:
+            f.write(text)
+        print(f"Written to {parsed.out}")
+    else:
+        print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run_export_context(sys.argv[1:]))

mygraph/extractor.py

@@ -0,0 +1,243 @@
+"""
+extractor.py — Stage 1 of the v1 ingest pipeline.
+
+Reads a markdown file, calls the Anthropic API with a schema-constrained prompt,
+and writes a candidates.json. No graph mutation here.
+
+Provenance-or-bust: the prompt requires literal excerpts for `high`-confidence
+candidates. Validator (Stage 2) enforces it; this stage just asks for it.
+
+Env:
+    Anthropic auth is inferred from env. Supported providers:
+      - Anthropic API: ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN
+      - Foundry: ANTHROPIC_FOUNDRY_API_KEY plus resource/base URL
+      - Bedrock: AWS_BEARER_TOKEN_BEDROCK or AWS credentials plus region
+    MYGRAPH_MODEL — optional model override; default `claude-sonnet-4-6`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from mygraph import Graph, NODE_TYPES, EDGE_TYPES, slug
+try:
+    from .anthropic_client import get_anthropic_client
+except ImportError:  # direct script execution
+    from anthropic_client import get_anthropic_client
+
+DEFAULT_MODEL = "claude-sonnet-4-6"
+
+EXTRACTION_TOOL = {
+    "name": "emit_candidates",
+    "description": "Emit candidate nodes and edges extracted from the source markdown.",
+    "input_schema": {
+        "type": "object",
+        "properties": {
+            "source": {
+                "type": "object",
+                "properties": {
+                    "id": {"type": "string"},
+                    "label": {"type": "string"},
+                    "body": {"type": "string"},
+                },
+                "required": ["id", "label", "body"],
+            },
+            "nodes": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "id": {"type": "string"},
+                        "type": {"type": "string", "enum": sorted(NODE_TYPES)},
+                        "label": {"type": "string"},
+                        "body": {"type": "string"},
+                        "confidence": {"type": "string", "enum": ["high", "medium", "low"]},
+                        "excerpt": {"type": "string"},
+                    },
+                    "required": ["id", "type", "label", "confidence"],
+                },
+            },
+            "edges": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "src": {"type": "string"},
+                        "dst": {"type": "string"},
+                        "type": {"type": "string", "enum": sorted(EDGE_TYPES)},
+                        "confidence": {"type": "string", "enum": ["high", "medium", "low"]},
+                        "excerpt": {"type": "string"},
+                    },
+                    "required": ["src", "dst", "type", "confidence"],
+                },
+            },
+        },
+        "required": ["source", "nodes", "edges"],
+    },
+}
+
+
+PROMPT_TEMPLATE = """\
+You are extracting nodes and edges for a personal knowledge graph centered on the user.
+The graph stores durable concepts (Person, Idea, Project, Goal, Topic, Reference,
+Question, Decision, Source) and relations between them.
+
+Rules:
+1. Every node and edge MUST cite a literal excerpt from the source. No paraphrase.
+2. Use confidence "high" only when you have a direct quote in the `excerpt` field.
+3. Use confidence "medium" for clear paraphrase (still quote what you paraphrased FROM).
+4. Use confidence "low" for inference. Quote what you inferred FROM.
+5. Slug-style IDs: lowercase, hyphenated, type-prefixed. E.g. `idea:context-memory`.
+6. Reuse existing IDs (below) when a candidate refers to an existing concept.
+7. Do NOT invent biographical facts. If the source doesn't say it, it doesn't go in.
+8. The Source node `id` MUST equal: {source_id}
+9. Every NEW concept node MUST have a `MENTIONED_IN` edge to the Source.
+10. Output the tool call exactly per schema. No prose.
+
+Allowed node types: {node_types}
+Allowed edge types: {edge_types}
+
+Existing node IDs (reuse when applicable):
+{existing_ids}
+
+SOURCE METADATA:
+  id    : {source_id}
+  label : {source_label}
+  path  : {source_path}
+
+SOURCE MARKDOWN follows between <<<SOURCE>>> markers. Extract.
+
+<<<SOURCE>>>
+{source_text}
+<<<SOURCE>>>
+"""
+
+
+def build_source_decl(md_path: Path) -> dict:
+    sid_slug = slug(md_path.stem)
+    return {
+        "source_id": f"source:{sid_slug}",
+        "source_label": md_path.name,
+        "source_path": str(md_path.resolve()),
+        "ingested_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+def call_anthropic(prompt: str, model: str | None = None) -> dict:
+    """Invoke Claude with the extraction tool. Returns the tool input dict."""
+    try:
+        client, config = get_anthropic_client()
+    except RuntimeError as e:
+        raise SystemExit(
+            f"extractor: {e}\n"
+            "Or run `mykg ingest <file> --candidates-file <path>` "
+            "to skip extraction with a hand-curated candidates JSON."
+        ) from e
+    model = model or config.model
+    resp = client.messages.create(
+        model=model,
+        max_tokens=8000,
+        tools=[EXTRACTION_TOOL],
+        tool_choice={"type": "tool", "name": "emit_candidates"},
+        messages=[{"role": "user", "content": prompt}],
+    )
+    for block in resp.content:
+        if getattr(block, "type", None) == "tool_use" and block.name == "emit_candidates":
+            return block.input  # type: ignore[return-value]
+    raise RuntimeError("extractor: model did not emit the emit_candidates tool call.")
+
+
+def ensure_provenance_edges(payload: dict) -> int:
+    """Backfill MENTIONED_IN edges when a gateway drops tool-emitted edges."""
+    src_id = payload.get("source", {}).get("id")
+    if not src_id:
+        return 0
+
+    edges = payload.get("edges")
+    if not isinstance(edges, list):
+        edges = []
+        payload["edges"] = edges
+    nodes = payload.get("nodes", [])
+    if not isinstance(nodes, list):
+        return 0
+
+    existing = {
+        (e.get("src"), e.get("dst"), e.get("type"))
+        for e in edges
+        if isinstance(e, dict)
+    }
+    injected = 0
+    for node in nodes:
+        if not isinstance(node, dict):
+            continue
+        node_id = node.get("id")
+        if not node_id or node_id == src_id:
+            continue
+        key = (node_id, src_id, "MENTIONED_IN")
+        if key in existing:
+            continue
+        edges.append({
+            "src": node_id,
+            "dst": src_id,
+            "type": "MENTIONED_IN",
+            "confidence": node.get("confidence", "medium"),
+            "excerpt": (node.get("excerpt") or node.get("body") or "")[:300],
+        })
+        existing.add(key)
+        injected += 1
+    return injected
+
+
+def extract(md_path: Path, out_path: Path | None = None,
+            model: str | None = None) -> dict:
+    """End-to-end extract: read markdown, call LLM, write candidates.json."""
+    g = Graph.load()
+    decl = build_source_decl(md_path)
+    source_text = md_path.read_text(encoding="utf-8")
+    existing_ids = sorted(g.nodes.keys())
+    prompt = PROMPT_TEMPLATE.format(
+        source_id=decl["source_id"],
+        source_label=decl["source_label"],
+        source_path=decl["source_path"],
+        node_types=", ".join(sorted(NODE_TYPES)),
+        edge_types=", ".join(sorted(EDGE_TYPES)),
+        existing_ids="\n".join(f"  - {i}" for i in existing_ids),
+        source_text=source_text,
+    )
+    payload = call_anthropic(prompt, model=model)
+    injected = ensure_provenance_edges(payload)
+    if injected:
+        print(
+            "extractor: gateway returned missing provenance edges; "
+            f"synthesized {injected} MENTIONED_IN edges.",
+            file=sys.stderr,
+        )
+    # ensure source_path/ingested_at hitch a ride for downstream stages
+    payload.setdefault("_meta", {})
+    payload["_meta"]["source_path"] = decl["source_path"]
+    payload["_meta"]["ingested_at"] = decl["ingested_at"]
+    payload["_meta"]["model"] = model or os.environ.get("MYGRAPH_MODEL", DEFAULT_MODEL)
+    if out_path:
+        out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    return payload
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        print("Usage: python extractor.py <path/to/file.md> [out.json]")
+        return 1
+    md = Path(argv[1]).expanduser().resolve()
+    out = Path(argv[2]).resolve() if len(argv) > 2 else md.parent / f"{md.stem}.candidates.json"
+    payload = extract(md, out)
+    print(f"extractor: wrote {len(payload.get('nodes', []))} nodes, "
+          f"{len(payload.get('edges', []))} edges -> {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

mygraph/extractor_openai.py

@@ -0,0 +1,165 @@
+"""
+extractor_openai.py — Stage 1 extractor backed by OpenAI Responses API.
+
+Reads a markdown file, calls OpenAI with a JSON-schema constrained response,
+and writes a candidates.json. No graph mutation here.
+
+Env:
+    OPENAI_API_KEY        — required.
+    MYGRAPH_OPENAI_MODEL  — optional model override; default `gpt-5.2`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from mygraph import Graph, NODE_TYPES, EDGE_TYPES
+
+try:
+    from .extractor import (
+        EXTRACTION_TOOL,
+        PROMPT_TEMPLATE,
+        build_source_decl,
+        ensure_provenance_edges,
+    )
+except ImportError:  # direct script execution
+    from extractor import (
+        EXTRACTION_TOOL,
+        PROMPT_TEMPLATE,
+        build_source_decl,
+        ensure_provenance_edges,
+    )
+
+
+DEFAULT_MODEL = os.environ.get("MYGRAPH_OPENAI_MODEL", "gpt-5.2")
+
+
+def _response_text(resp: Any) -> str:
+    """Extract text from OpenAI Responses SDK objects across SDK versions."""
+    output_text = getattr(resp, "output_text", None)
+    if output_text:
+        return output_text
+
+    chunks: list[str] = []
+    for item in getattr(resp, "output", []) or []:
+        for content in getattr(item, "content", []) or []:
+            text = getattr(content, "text", None)
+            if text:
+                chunks.append(text)
+    return "".join(chunks)
+
+
+def _loads_json(text: str) -> dict:
+    stripped = text.strip()
+    if stripped.startswith("```"):
+        stripped = stripped.strip("`")
+        stripped = stripped.split("\n", 1)[1] if "\n" in stripped else stripped
+        stripped = stripped.rsplit("```", 1)[0] if stripped.endswith("```") else stripped
+    return json.loads(stripped)
+
+
+def call_openai(prompt: str, model: str = DEFAULT_MODEL) -> dict:
+    """Invoke OpenAI with Structured Outputs. Returns the parsed JSON dict."""
+    try:
+        from openai import OpenAI  # type: ignore
+    except ImportError as e:
+        raise SystemExit(
+            "extractor_openai: `openai` package not installed. Run:\n"
+            "    python -m pip install -e '.[openai]'"
+        ) from e
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise SystemExit(
+            "extractor_openai: OPENAI_API_KEY env var is not set.\n"
+            "Either export it, or run `mykg ingest <file> --candidates-file <path>` "
+            "to skip extraction with a hand-curated candidates JSON."
+        )
+
+    client = OpenAI()
+    resp = client.responses.create(
+        model=model,
+        input=[{"role": "user", "content": prompt}],
+        max_output_tokens=8000,
+        text={
+            "format": {
+                "type": "json_schema",
+                "name": EXTRACTION_TOOL["name"],
+                "description": EXTRACTION_TOOL["description"],
+                "schema": EXTRACTION_TOOL["input_schema"],
+                "strict": False,
+            }
+        },
+    )
+    text = _response_text(resp)
+    if not text:
+        raise RuntimeError("extractor_openai: model returned no text output.")
+    try:
+        return _loads_json(text)
+    except json.JSONDecodeError as e:
+        raise RuntimeError(
+            f"extractor_openai: model returned non-JSON: {text[:300]}"
+        ) from e
+
+
+def extract(md_path: Path, out_path: Path | None = None,
+            model: str = DEFAULT_MODEL) -> dict:
+    """End-to-end extract via OpenAI. Same return shape as extractor.extract."""
+    g = Graph.load()
+    decl = build_source_decl(md_path)
+    source_text = md_path.read_text(encoding="utf-8")
+    existing_ids = sorted(g.nodes.keys())
+    prompt = PROMPT_TEMPLATE.format(
+        source_id=decl["source_id"],
+        source_label=decl["source_label"],
+        source_path=decl["source_path"],
+        node_types=", ".join(sorted(NODE_TYPES)),
+        edge_types=", ".join(sorted(EDGE_TYPES)),
+        existing_ids="\n".join(f"  - {i}" for i in existing_ids),
+        source_text=source_text,
+    )
+    payload = call_openai(prompt, model=model)
+    injected = ensure_provenance_edges(payload)
+    if injected:
+        print(
+            "extractor_openai: gateway returned missing provenance edges; "
+            f"synthesized {injected} MENTIONED_IN edges.",
+            file=sys.stderr,
+        )
+    payload.setdefault("_meta", {})
+    payload["_meta"]["source_path"] = decl["source_path"]
+    payload["_meta"]["ingested_at"] = decl["ingested_at"]
+    payload["_meta"]["model"] = model
+    payload["_meta"]["backend"] = "openai"
+    if out_path:
+        out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    return payload
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        print("Usage: python extractor_openai.py <path/to/file.md> [out.json] [--model NAME]")
+        return 1
+    md = Path(argv[1]).expanduser().resolve()
+    out = None
+    model = DEFAULT_MODEL
+    rest = argv[2:]
+    if "--model" in rest:
+        i = rest.index("--model")
+        model = rest[i + 1]
+        del rest[i:i + 2]
+    if rest:
+        out = Path(rest[0]).expanduser().resolve()
+    else:
+        out = md.parent / f"{md.stem}.candidates.openai.json"
+    payload = extract(md, out, model=model)
+    print(f"extractor_openai: model={model}  "
+          f"nodes={len(payload.get('nodes', []))}  "
+          f"edges={len(payload.get('edges', []))}  ->  {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))