know-do-graph 0.1.0__py3-none-any.whl

agents/review_agent/tools.py

@@ -0,0 +1,472 @@
+"""Tools for the ReviewAgent.
+
+The review agent uses these tools to examine the graph incrementally —
+it never receives the full graph dump at once.  Instead it:
+  - picks under-reviewed nodes (weighted toward low review_count)
+  - inspects a node's full details and its local neighbourhood
+  - updates review/modify counters after each inspection
+  - proposes and applies targeted fixes (title, tags, aliases)
+"""
+
+from __future__ import annotations
+
+import random
+from datetime import datetime, timezone
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Sampling / overview
+# ---------------------------------------------------------------------------
+
+
+def sample_nodes_for_review(batch_size: int = 5, graph: Any = None) -> list[dict]:
+    """Return a weighted-random sample of nodes, preferring those with low review_count.
+
+    Nodes with fewer reviews are much more likely to be selected so the agent
+    makes forward progress on unchecked parts of the graph.
+
+    Returns id, slug, title, type, tags, aliases, review_count, modify_count.
+    """
+    from core import app_state
+    from core.retrieval.retrieval import RetrievalEngine
+    from core.storage.database import SessionLocal
+
+    g = graph or app_state.graph
+    with SessionLocal() as db:
+        engine = RetrievalEngine(db, g)
+        all_entries = engine.list_entries(limit=5000)
+
+    if not all_entries:
+        return []
+
+    # Weight = 1 / (review_count + 1)  → unseen nodes are most likely
+    weights = [1.0 / (e.metadata.review_count + 1) for e in all_entries]
+    k = min(batch_size, len(all_entries))
+    selected = random.choices(all_entries, weights=weights, k=k)
+    # Deduplicate while preserving order
+    seen_ids: set[str] = set()
+    unique: list = []
+    for e in selected:
+        if e.id not in seen_ids:
+            seen_ids.add(e.id)
+            unique.append(e)
+
+    return [
+        {
+            "id": e.id,
+            "slug": e.slug,
+            "title": e.title,
+            "type": e.entry_type.value,
+            "tags": e.tags,
+            "aliases": e.aliases,
+            "review_count": e.metadata.review_count,
+            "modify_count": e.metadata.modify_count,
+        }
+        for e in unique
+    ]
+
+
+def get_graph_summary(graph: Any = None) -> dict:
+    """Return aggregate statistics useful for high-level review.
+
+    Includes node/edge counts, type distribution, and review coverage
+    (how many nodes have been reviewed at least once).
+    """
+    from collections import Counter
+
+    from core import app_state
+    from core.retrieval.retrieval import RetrievalEngine
+    from core.storage.database import SessionLocal
+
+    g = graph or app_state.graph
+    stats = g.stats()
+
+    with SessionLocal() as db:
+        engine = RetrievalEngine(db, g)
+        all_entries = engine.list_entries(limit=5000)
+
+    type_dist = dict(Counter(e.entry_type.value for e in all_entries))
+    reviewed = sum(1 for e in all_entries if e.metadata.review_count > 0)
+    total = len(all_entries)
+
+    return {
+        "stats": stats,
+        "type_distribution": type_dist,
+        "total_nodes": total,
+        "reviewed_nodes": reviewed,
+        "unreviewed_nodes": total - reviewed,
+        "review_coverage_pct": round(100 * reviewed / total, 1) if total else 0,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Node inspection
+# ---------------------------------------------------------------------------
+
+
+def inspect_node(identifier: str, graph: Any = None) -> dict:
+    """Retrieve full details of a node including review metadata and local edges.
+
+    Returns title, type, tags, aliases, content (first 800 chars), refs,
+    review_count, modify_count, and a list of neighbouring node titles.
+    """
+    from core import app_state
+    from core.retrieval.retrieval import RetrievalEngine
+    from core.storage.database import SessionLocal
+
+    g = graph or app_state.graph
+    with SessionLocal() as db:
+        engine = RetrievalEngine(db, g)
+        entry = engine.resolve_identifier(identifier)
+        if entry is None:
+            return {"error": f"Entry '{identifier}' not found."}
+
+        neighbors_raw = g.get_neighbors(entry.id, direction="both")
+        neighbor_details = []
+        for nbr in neighbors_raw:
+            nbr_entry = engine.get_entry_by_id(nbr["id"])
+            neighbor_details.append(
+                {
+                    "id": nbr["id"],
+                    "title": nbr_entry.title if nbr_entry else "?",
+                    "relation": nbr.get("relation"),
+                    "direction": nbr.get("direction"),
+                }
+            )
+
+    return {
+        "id": entry.id,
+        "slug": entry.slug,
+        "title": entry.title,
+        "type": entry.entry_type.value,
+        "tags": entry.tags,
+        "aliases": entry.aliases,
+        "content_preview": entry.content[:800],
+        "refs": entry.internal_refs,
+        "source": entry.metadata.source_provenance,
+        "status": entry.metadata.refinement_status.value,
+        "review_count": entry.metadata.review_count,
+        "modify_count": entry.metadata.modify_count,
+        "last_reviewed_at": entry.metadata.last_reviewed_at.isoformat() if entry.metadata.last_reviewed_at else None,
+        "neighbors": neighbor_details,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Review tracking
+# ---------------------------------------------------------------------------
+
+
+def mark_reviewed(entry_id: str, was_modified: bool = False, graph: Any = None) -> dict:
+    """Increment review_count (and optionally modify_count) on an entry.
+
+    Call this after inspecting a node, regardless of whether changes were made.
+    Pass was_modified=True if you also edited the node in this review pass.
+    """
+    from core import app_state
+    from core.retrieval.retrieval import RetrievalEngine
+    from core.storage.database import SessionLocal
+    from core.storage.repository import EntryRepository
+
+    g = graph or app_state.graph
+    with SessionLocal() as db:
+        engine = RetrievalEngine(db, g)
+        entry = engine.get_entry_by_id(entry_id)
+        if entry is None:
+            return {"error": f"Entry '{entry_id}' not found."}
+
+        entry.metadata.review_count += 1
+        entry.metadata.last_reviewed_at = datetime.now(timezone.utc)
+        if was_modified:
+            entry.metadata.modify_count += 1
+
+        EntryRepository(db).update(entry)
+
+    return {
+        "entry_id": entry_id,
+        "review_count": entry.metadata.review_count,
+        "modify_count": entry.metadata.modify_count,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Cleaning helpers (re-exported from graph_agent.tools for convenience)
+# ---------------------------------------------------------------------------
+
+
+def update_entry(
+    entry_id: str,
+    title: str | None = None,
+    content: str | None = None,
+    entry_type: str | None = None,
+    tags: list[str] | None = None,
+    aliases: list[str] | None = None,
+    graph: Any = None,
+) -> dict:
+    """Update fields on an existing entry and bump modify_count."""
+    from agents.graph_agent.tools import update_entry as _update_entry
+
+    result = _update_entry(
+        entry_id=entry_id,
+        title=title,
+        content=content,
+        entry_type=entry_type,
+        tags=tags,
+        aliases=aliases,
+        graph=graph,
+    )
+    if "error" not in result:
+        mark_reviewed(result["id"], was_modified=True, graph=graph)
+    return result
+
+
+def merge_entries(
+    primary_id: str,
+    duplicate_id: str,
+    merge_aliases: bool = True,
+    merge_tags: bool = True,
+    graph: Any = None,
+) -> dict:
+    """Merge duplicate into primary, then mark primary as modified."""
+    from agents.graph_agent.tools import merge_entries as _merge_entries
+
+    result = _merge_entries(
+        primary_id=primary_id,
+        duplicate_id=duplicate_id,
+        merge_aliases=merge_aliases,
+        merge_tags=merge_tags,
+        graph=graph,
+    )
+    if result.get("merged"):
+        mark_reviewed(result["primary_id"], was_modified=True, graph=graph)
+    return result
+
+
+def search_entries(query: str, limit: int = 10, mode: str = "hybrid", graph: Any = None) -> list[dict]:
+    """Hybrid semantic + keyword search — use to find duplicate or related candidates."""
+    from agents.graph_agent.tools import search_entries as _search_entries
+
+    return _search_entries(query=query, limit=limit, mode=mode, graph=graph)
+
+
+def create_edge(
+    source_id: str,
+    target_id: str,
+    relation: str = "related_to",
+    weight: float = 1.0,
+    graph: Any = None,
+) -> dict:
+    """Create an edge between two entries."""
+    from agents.graph_agent.tools import create_edge as _create_edge
+
+    return _create_edge(source_id=source_id, target_id=target_id, relation=relation, weight=weight, graph=graph)
+
+
+def delete_edge(edge_id: str, graph: Any = None) -> dict:
+    """Delete an edge by ID."""
+    from agents.graph_agent.tools import delete_edge as _delete_edge
+
+    return _delete_edge(edge_id=edge_id, graph=graph)
+
+
+# ---------------------------------------------------------------------------
+# OpenAI tool schemas
+# ---------------------------------------------------------------------------
+
+REVIEW_TOOL_SCHEMAS: list[dict] = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_graph_summary",
+            "description": (
+                "Get high-level graph statistics including node count, type distribution, "
+                "and review coverage. Use at the start of each review session."
+            ),
+            "parameters": {"type": "object", "properties": {}},
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "sample_nodes_for_review",
+            "description": (
+                "Return a weighted-random batch of nodes to review. "
+                "Nodes with fewer prior reviews are selected more often. "
+                "Use this to pick the next set of nodes to inspect."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "batch_size": {"type": "integer", "default": 5, "description": "Number of nodes to sample"},
+                },
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "inspect_node",
+            "description": (
+                "Get full details of a node: title, type, tags, aliases, content preview, "
+                "review history, and neighbouring nodes with relation types. "
+                "Always inspect a node before deciding to modify it."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "identifier": {"type": "string", "description": "Node ID or slug"},
+                },
+                "required": ["identifier"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "mark_reviewed",
+            "description": (
+                "Record that you have reviewed a node. "
+                "Must be called for every node you inspect, even if no changes were made. "
+                "Set was_modified=True if you also edited the node."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "entry_id": {"type": "string"},
+                    "was_modified": {"type": "boolean", "default": False},
+                },
+                "required": ["entry_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "update_entry",
+            "description": (
+                "Update title, tags, aliases, content, or type of a node. "
+                "Use to: fix titles containing parenthetical acronyms (move acronym to aliases), "
+                "normalise tags to lowercase-hyphenated, remove redundant prefixes from titles, "
+                "or correct the entry_type."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "entry_id": {"type": "string", "description": "Node ID or slug"},
+                    "title": {"type": "string"},
+                    "content": {"type": "string"},
+                    "entry_type": {
+                        "type": "string",
+                        "enum": ["capability", "procedure", "workflow", "tool", "repository",
+                                 "environment", "dependency", "data", "analytical", "memory", "generic"],
+                    },
+                    "tags": {"type": "array", "items": {"type": "string"}},
+                    "aliases": {"type": "array", "items": {"type": "string"}},
+                },
+                "required": ["entry_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "search_entries",
+            "description": (
+                "Search for nodes using hybrid semantic + keyword retrieval. "
+                "The default 'hybrid' mode combines embedding-based vector similarity with "
+                "keyword scoring (RRF fusion). Use 'semantic' to find conceptually related nodes "
+                "even when wording differs — ideal for surfacing near-duplicate candidates. "
+                "Use 'keyword' for exact title or acronym lookups. "
+                "If a search misses, retry with a different mode or a broader/rephrased query."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {"type": "string"},
+                    "limit": {"type": "integer", "default": 10},
+                    "mode": {
+                        "type": "string",
+                        "enum": ["hybrid", "semantic", "keyword"],
+                        "default": "hybrid",
+                        "description": (
+                            "hybrid: keyword + embedding ANN fused (default). "
+                            "semantic: embedding-only, best for conceptual/paraphrase matching. "
+                            "keyword: exact text match, best for known titles or acronyms."
+                        ),
+                    },
+                },
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "merge_entries",
+            "description": (
+                "Merge a duplicate node into a primary node. "
+                "Re-targets edges, merges aliases/tags, deletes the duplicate."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "primary_id": {"type": "string"},
+                    "duplicate_id": {"type": "string"},
+                    "merge_aliases": {"type": "boolean", "default": True},
+                    "merge_tags": {"type": "boolean", "default": True},
+                },
+                "required": ["primary_id", "duplicate_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "create_edge",
+            "description": "Add a typed edge between two nodes when a relationship is missing.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "source_id": {"type": "string"},
+                    "target_id": {"type": "string"},
+                    "relation": {
+                        "type": "string",
+                        "enum": ["dependency", "compatible_with", "alternative_to", "related_workflow",
+                                 "generated_from", "memory_of", "refinement_of", "derived_from",
+                                 "warning_about", "cited_by", "wikilink", "prerequisite", "replacement",
+                                 "execution_pathway", "transformation", "provenance", "compatibility"],
+                    },
+                    "weight": {"type": "number", "default": 1.0},
+                },
+                "required": ["source_id", "target_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "delete_edge",
+            "description": "Delete a redundant or incorrect edge by its ID.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "edge_id": {"type": "string"},
+                },
+                "required": ["edge_id"],
+            },
+        },
+    },
+]
+
+REVIEW_TOOL_DISPATCH: dict[str, Any] = {
+    "get_graph_summary": get_graph_summary,
+    "sample_nodes_for_review": sample_nodes_for_review,
+    "inspect_node": inspect_node,
+    "mark_reviewed": mark_reviewed,
+    "update_entry": update_entry,
+    "search_entries": search_entries,
+    "merge_entries": merge_entries,
+    "create_edge": create_edge,
+    "delete_edge": delete_edge,
+}

api/main.py

@@ -0,0 +1,136 @@
+"""Know-Do Graph — FastAPI application."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.staticfiles import StaticFiles
+
+from fastapi import Request
+from fastapi.responses import PlainTextResponse
+
+from api.routes import entries, graph as graph_routes, mem as mem_routes, agent as agent_routes
+from api.routes import remote as remote_routes
+from api.routes import remote_sync as remote_sync_routes
+from api.routes import retrieve as retrieve_routes
+from core.app_state import graph
+from core.storage.database import SessionLocal, init_db
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Initialise the database and rebuild the in-memory graph on startup."""
+    init_db()
+    from core import events as _events
+    _events.set_loop(asyncio.get_running_loop())
+    from core.sync.db_watcher import reload_graph_from_db
+
+    reload_graph_from_db(graph)
+
+    sync_task: asyncio.Task | None = None
+    if os.environ.get("KDG_REMOTE_SYNC_ENABLED", "").lower() in ("1", "true", "yes", "on"):
+        interval = int(os.environ.get("KDG_REMOTE_SYNC_INTERVAL_SECONDS", "900"))
+        from core.sync.remote_sync import run_periodic_sync
+
+        sync_task = asyncio.create_task(run_periodic_sync(interval))
+        logger.info("remote-sync background loop started (interval=%ss)", interval)
+
+    # DB-change watcher: detects mutations written by out-of-process CLI commands
+    # (e.g. `python main.py extract …`) and refreshes the in-memory graph + SSE.
+    watcher_task: asyncio.Task | None = None
+    watch_interval = int(os.environ.get("KDG_DB_WATCH_INTERVAL_SECONDS", "3"))
+    if watch_interval > 0:
+        from core.sync.db_watcher import run_db_watcher
+
+        watcher_task = asyncio.create_task(run_db_watcher(graph, watch_interval))
+        logger.info("db-watcher started (interval=%ss)", watch_interval)
+
+    try:
+        yield
+    finally:
+        from core import events as _events
+        _events.signal_shutdown()
+        for task in (sync_task, watcher_task):
+            if task is not None:
+                task.cancel()
+                try:
+                    await task
+                except (asyncio.CancelledError, Exception):
+                    pass
+
+
+app = FastAPI(
+    title="Know-Do Graph API",
+    description=(
+        "Agent-facing interface for a wiki-native executable knowledge graph. "
+        "Search entries, traverse relations, and navigate operational knowledge."
+    ),
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(entries.router, prefix="/entries", tags=["entries"])
+app.include_router(graph_routes.router, prefix="/graph", tags=["graph"])
+app.include_router(mem_routes.router, prefix="/mem", tags=["mem"])
+app.include_router(agent_routes.router, prefix="/agent", tags=["agent"])
+app.include_router(remote_routes.router, prefix="/remote", tags=["remote"])
+app.include_router(remote_sync_routes.router, prefix="/remote-sync", tags=["remote-sync"])
+app.include_router(retrieve_routes.router, prefix="/retrieve", tags=["retrieve"])
+
+
+@app.get("/health", tags=["meta"])
+def health() -> dict:
+    return {"status": "ok", **graph.stats()}
+
+
+@app.get("/", response_class=PlainTextResponse, include_in_schema=False)
+def root_instructions(request: Request) -> PlainTextResponse:
+    """Return the plain-text instruction sheet for any client that hits the server root."""
+    from api.routes.remote import _render_instructions
+    return PlainTextResponse(_render_instructions(request))
+
+
+# ── Frontend ──────────────────────────────────────────────────────────────────
+# After `npm run build` in frontend/, Vite emits frontend/dist/ with relative
+# asset URLs (vite.config.js: base: './'). We serve dist/index.html at /ui and
+# the hashed bundle at /assets. In dev, prefer `npm run dev` (Vite on :5173
+# with API proxy) — direct HMR, this mount unused.
+_FRONTEND_ROOT = Path(__file__).parent.parent / "frontend"
+_FRONTEND_DIST = _FRONTEND_ROOT / "dist"
+
+if _FRONTEND_DIST.is_dir() and (_FRONTEND_DIST / "index.html").is_file():
+    if (_FRONTEND_DIST / "assets").is_dir():
+        app.mount("/assets", StaticFiles(directory=str(_FRONTEND_DIST / "assets")), name="ui-assets")
+
+    @app.get("/ui", include_in_schema=False)
+    def serve_ui() -> FileResponse:
+        return FileResponse(str(_FRONTEND_DIST / "index.html"))
+
+else:
+    from fastapi.responses import HTMLResponse
+
+    @app.get("/ui", include_in_schema=False)
+    def serve_ui_not_built() -> HTMLResponse:
+        return HTMLResponse(
+            "<h1 style='font-family:sans-serif'>Frontend not built</h1>"
+            "<p style='font-family:sans-serif'>Run the following then restart the server:</p>"
+            "<pre style='background:#111;color:#0f0;padding:1em;border-radius:4px'>"
+            "cd frontend\nnpm install\nnpm run build</pre>",
+            status_code=503,
+        )

api/routes/agent.py

@@ -0,0 +1,81 @@
+"""Agent-chat routes (GraphAgent and ReviewAgent)."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from core.app_state import graph as _graph
+
+router = APIRouter()
+
+
+class ChatRequest(BaseModel):
+    message: str
+    model: str | None = None
+
+
+class ReviewRequest(BaseModel):
+    instructions: str = ""
+    batch_size: int = 5
+    model: str | None = None
+
+
+@router.post("/graph/chat", tags=["agent"])
+def graph_agent_chat(body: ChatRequest) -> dict:
+    """Send a message to the GraphAgent and receive its response."""
+    import os
+
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise HTTPException(status_code=503, detail="OPENAI_API_KEY not configured.")
+
+    from agents.graph_agent.agent import GraphAgent
+
+    agent = GraphAgent(graph=_graph, model=body.model)
+    response = agent.chat(body.message)
+    return {"response": response}
+
+
+@router.post("/review/run", tags=["agent"])
+def review_agent_run(body: ReviewRequest) -> dict:
+    """Run one review session with the ReviewAgent."""
+    import os
+
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise HTTPException(status_code=503, detail="OPENAI_API_KEY not configured.")
+
+    from agents.review_agent.agent import ReviewAgent
+
+    agent = ReviewAgent(graph=_graph, model=body.model, batch_size=body.batch_size)
+    response = agent.run_review(instructions=body.instructions)
+    return {"response": response}
+
+
+@router.post("/review/chat", tags=["agent"])
+def review_agent_chat(body: ChatRequest) -> dict:
+    """Send a single message to the ReviewAgent."""
+    import os
+
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise HTTPException(status_code=503, detail="OPENAI_API_KEY not configured.")
+
+    from agents.review_agent.agent import ReviewAgent
+
+    agent = ReviewAgent(graph=_graph, model=body.model)
+    response = agent.chat(body.message)
+    return {"response": response}
+
+
+@router.post("/orchestrate", tags=["agent"])
+def orchestrate(body: ChatRequest) -> dict:
+    """Route a request through the Orchestrator to the appropriate agent(s)."""
+    import os
+
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise HTTPException(status_code=503, detail="OPENAI_API_KEY not configured.")
+
+    from agents.orchestrator.agent import OrchestratorAgent
+
+    agent = OrchestratorAgent(graph=_graph, model=body.model)
+    response = agent.chat(body.message)
+    return {"response": response}