know-do-graph 0.1.0__py3-none-any.whl

agents/maintenance_agent/agent.py

@@ -0,0 +1,283 @@
+"""Maintenance agent.
+
+Performs routine graph health and consistency tasks:
+- Remove dangling edges pointing to deleted entries
+- Rebuild the in-memory graph from the database
+- Export entries to YAML node files
+- Promote Mem-Graph traces into full Know-Do Graph entries
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from core.graph.graph import KnowDoGraph
+from core.schemas.entry import Entry, EntryMetadata, EntryType, RefinementStatus
+from core.storage.database import SessionLocal
+from core.storage.repository import EdgeRepository, EntryRepository
+
+
+class MaintenanceAgent:
+    def __init__(self, graph: KnowDoGraph) -> None:
+        self._graph = graph
+
+    def remove_dangling_edges(self) -> int:
+        """Delete edges whose source or target entry no longer exists."""
+        removed = 0
+        with SessionLocal() as db:
+            entry_repo = EntryRepository(db)
+            edge_repo = EdgeRepository(db)
+            entry_ids = {e.id for e in entry_repo.get_all()}
+            for edge in edge_repo.get_all():
+                if edge.source_id not in entry_ids or edge.target_id not in entry_ids:
+                    edge_repo.delete(edge.id)
+                    self._graph.remove_edge(edge.source_id, edge.target_id)
+                    removed += 1
+        return removed
+
+    def rebuild_graph(self) -> None:
+        """Rebuild the in-memory graph from the current database state."""
+        with SessionLocal() as db:
+            entries = EntryRepository(db).get_all()
+            edges = EdgeRepository(db).get_all()
+        self._graph.rebuild_from_db(entries, edges)
+
+    def export_to_yaml(self, output_dir: Path) -> int:
+        """Write each entry as a YAML file under *output_dir*.
+
+        Returns the number of files written.
+        """
+        import yaml  # pyyaml
+
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        with SessionLocal() as db:
+            entries = EntryRepository(db).get_all()
+        for entry in entries:
+            data = entry.model_dump(mode="json")
+            file_path = output_dir / f"{entry.slug}.yaml"
+            file_path.write_text(
+                yaml.dump(data, allow_unicode=True, sort_keys=False),
+                encoding="utf-8",
+            )
+        return len(entries)
+
+    def promote_mem_entry(
+        self,
+        mem_id: str,
+        session_id: str = "default",
+        entry_type: EntryType = EntryType.memory,
+        tags: Optional[list[str]] = None,
+    ) -> Optional[Entry]:
+        """Promote a Mem-Graph trace into a full Know-Do Graph entry."""
+        from core.memory.memgraph import MemGraph
+        from core.storage.repository import EntryRepository
+
+        mg = MemGraph(session_id)
+        mem_entry = mg.get(mem_id)
+        if not mem_entry:
+            return None
+
+        entry = Entry(
+            title=f"Memory: {mem_entry.content[:60]}",
+            entry_type=entry_type,
+            content=mem_entry.content,
+            tags=(tags or []) + mem_entry.tags,
+            metadata=EntryMetadata(
+                source_provenance=f"mem-graph:{session_id}:{mem_id}",
+                extraction_method="mem_promotion",
+                refinement_status=RefinementStatus.raw,
+            ),
+        )
+
+        with SessionLocal() as db:
+            saved = EntryRepository(db).create(entry)
+        self._graph.add_entry(saved)
+        mg.mark_promoted(mem_id, saved.id)
+        return saved
+
+    # ------------------------------------------------------------------
+    # Query helpers — surface entries that need attention
+    # ------------------------------------------------------------------
+
+    def list_unverified(self, limit: int = 100) -> list[Entry]:
+        """Return entries whose verification_status is 'unverified'."""
+        from core.schemas.entry import VerificationStatus
+
+        with SessionLocal() as db:
+            entries = EntryRepository(db).get_all()
+        return [
+            e for e in entries
+            if e.metadata.verification_status == VerificationStatus.unverified
+        ][:limit]
+
+    def list_bugged(self, limit: int = 100) -> list[Entry]:
+        """Return entries flagged as bugged via feedback."""
+        from core.schemas.entry import VerificationStatus
+
+        with SessionLocal() as db:
+            entries = EntryRepository(db).get_all()
+        return [
+            e for e in entries
+            if e.metadata.verification_status == VerificationStatus.bugged
+        ][:limit]
+
+    def list_needs_generalization(self, limit: int = 100) -> list[Entry]:
+        """Return entries flagged by the abstraction check."""
+        with SessionLocal() as db:
+            entries = EntryRepository(db).get_all()
+        return [e for e in entries if e.metadata.needs_generalization][:limit]
+
+    # ------------------------------------------------------------------
+    # Hierarchical-memory maintenance
+    # ------------------------------------------------------------------
+
+    def extract_heuristics_from_node(
+        self,
+        entry_id: str,
+        dry_run: bool = True,
+    ) -> dict:
+        """Best-effort split of a flat skill blob into L3 / L4 child nodes.
+
+        Scans the body of *entry_id* for headed sections whose titles match
+        common heuristic / constraint patterns (case-insensitive substrings):
+
+        ============================ ===============
+        Heading contains             Maps to
+        ============================ ===============
+        "heuristic", "rule of thumb" L3 heuristic
+        "tips", "best practice"      L3 heuristic
+        "limitation", "failure"      L4 constraint
+        "caveat", "warning", "pitfall" L4 constraint
+        "do not use", "not suitable" L4 constraint
+        ============================ ===============
+
+        Non-destructive: the source node is not modified; child nodes are only
+        created when ``dry_run=False``. Returns a summary dict describing what
+        was (or would be) extracted.
+        """
+        import re
+
+        from core.retrieval.retrieval import RetrievalEngine
+        from core.schemas.edge import Edge, EdgeRelation
+        from core.schemas.entry import (
+            Entry,
+            EntryMetadata,
+            EntryType,
+            SkillLevel,
+        )
+
+        l3_keywords = ("heuristic", "rule of thumb", "tips", "best practice", "tip:", "guideline")
+        l4_keywords = (
+            "limitation", "failure", "caveat", "warning", "pitfall",
+            "do not use", "not suitable", "unsuitable", "instability", "known issue",
+        )
+
+        with SessionLocal() as db:
+            engine = RetrievalEngine(db, self._graph)
+            entry = engine.resolve_identifier(entry_id)
+            if entry is None:
+                return {"error": f"Entry '{entry_id}' not found."}
+
+            sections = _split_markdown_sections(entry.content)
+            heuristics: list[dict] = []
+            constraints: list[dict] = []
+            for heading, body in sections:
+                lower = heading.lower()
+                if any(k in lower for k in l4_keywords):
+                    constraints.append({"title": heading.strip(), "content": body.strip()})
+                elif any(k in lower for k in l3_keywords):
+                    heuristics.append({"title": heading.strip(), "content": body.strip()})
+
+            created_h: list[dict] = []
+            created_c: list[dict] = []
+            if not dry_run:
+                edge_repo = EdgeRepository(db)
+                repo = EntryRepository(db)
+                for item in heuristics:
+                    child = Entry(
+                        title=f"{entry.title} — {item['title']}"[:200],
+                        content=item["content"],
+                        entry_type=EntryType.heuristic,
+                        tags=list(dict.fromkeys(entry.tags + ["heuristic"])),
+                        metadata=EntryMetadata(
+                            skill_level=SkillLevel.L3,
+                            source_provenance=f"extracted_from:{entry.slug}",
+                            applicability={"parent": entry.slug},
+                        ),
+                    )
+                    saved = repo.create(child)
+                    edge = Edge(
+                        source_id=saved.id,
+                        target_id=entry.id,
+                        relation=EdgeRelation.heuristic_for,
+                    )
+                    edge_repo.create(edge)
+                    self._graph.add_entry(saved)
+                    self._graph.add_edge(edge)
+                    created_h.append({"id": saved.id, "slug": saved.slug, "title": saved.title})
+
+                for item in constraints:
+                    child = Entry(
+                        title=f"{entry.title} — {item['title']}"[:200],
+                        content=item["content"],
+                        entry_type=EntryType.constraint,
+                        tags=list(dict.fromkeys(entry.tags + ["constraint", "failure-mode"])),
+                        metadata=EntryMetadata(
+                            skill_level=SkillLevel.L4,
+                            source_provenance=f"extracted_from:{entry.slug}",
+                            applicability={"parent": entry.slug},
+                        ),
+                    )
+                    saved = repo.create(child)
+                    edge = Edge(
+                        source_id=saved.id,
+                        target_id=entry.id,
+                        relation=EdgeRelation.constraint_on,
+                    )
+                    edge_repo.create(edge)
+                    self._graph.add_entry(saved)
+                    self._graph.add_edge(edge)
+                    created_c.append({"id": saved.id, "slug": saved.slug, "title": saved.title})
+
+                # Denormalise constraint slugs on parent.
+                if created_c:
+                    entry.metadata.failure_modes = list(
+                        dict.fromkeys(
+                            entry.metadata.failure_modes + [c["slug"] for c in created_c]
+                        )
+                    )
+                    repo.update(entry)
+
+        return {
+            "entry": entry.slug,
+            "dry_run": dry_run,
+            "candidate_heuristics": heuristics,
+            "candidate_constraints": constraints,
+            "created_heuristics": created_h,
+            "created_constraints": created_c,
+        }
+
+
+def _split_markdown_sections(text: str) -> list[tuple[str, str]]:
+    """Naive markdown splitter: returns list of (heading, body) pairs.
+
+    Recognises ``#``-style headings (any level). Text before the first heading
+    is associated with heading ``""``.
+    """
+    import re
+
+    sections: list[tuple[str, str]] = []
+    current_heading = ""
+    current_body: list[str] = []
+    for line in (text or "").splitlines():
+        m = re.match(r"^\s*#{1,6}\s+(.+?)\s*$", line)
+        if m:
+            sections.append((current_heading, "\n".join(current_body)))
+            current_heading = m.group(1)
+            current_body = []
+        else:
+            current_body.append(line)
+    sections.append((current_heading, "\n".join(current_body)))
+    return sections

agents/orchestrator/agent.py

@@ -0,0 +1,217 @@
+"""Orchestrator — routes a user request to the right agent or pipeline.
+
+The orchestrator is a thin LLM-driven router that:
+1. Reads the user's request (with optional graph context).
+2. Decides which agent(s) to invoke and with what parameters.
+3. Calls them in sequence or parallel and returns the combined result.
+
+Supported targets:
+- ``graph``  — GraphAgent (add/update/link knowledge)
+- ``review`` — ReviewAgent (audit & clean existing nodes)
+
+The orchestrator does NOT do graph work itself; it delegates entirely.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Callable
+
+from openai import OpenAI
+
+from core.graph.graph import KnowDoGraph
+
+_DEFAULT_MODEL = "qwen-plus"
+
+_SYSTEM_PROMPT = """You are the orchestrator for the Know-Do Graph system.
+Your only job is to decide which agent(s) should handle the user's request and
+call them with the right parameters.
+
+Available agents / pipelines
+-----------------------------
+1. ``run_graph_agent(message)``
+   Use for: adding new knowledge, searching/updating/linking nodes, enriching
+   content from the web, answering questions about graph content.
+
+2. ``run_review_agent(instructions, batch_size)``
+   Use for: auditing existing nodes, cleaning titles/tags/aliases, merging
+   duplicates, checking graph quality, improving coverage.
+   batch_size controls how many nodes are reviewed per session (default 5).
+
+Decision rules
+--------------
+- If the request is about *adding, updating, or querying* knowledge → graph agent.
+- If the request is about *cleaning, reviewing, fixing, auditing* the graph → review agent.
+- If both apply (e.g. "add new nodes and then review the related area") → call both in order.
+- If the intent is ambiguous, prefer the graph agent for content tasks and the
+  review agent for quality/maintenance tasks.
+
+Always call at least one agent. Never answer directly without delegating.
+After all agent calls are complete, give a concise summary of what was done.
+"""
+
+
+class OrchestratorAgent:
+    """Routes requests to GraphAgent and/or ReviewAgent.
+
+    Parameters
+    ----------
+    graph:
+        Shared KnowDoGraph instance.
+    model:
+        LLM model for the orchestrator's routing decision.
+    on_step:
+        Optional callback ``(event, data)`` forwarded to sub-agents for CLI display.
+    """
+
+    def __init__(
+        self,
+        graph: KnowDoGraph,
+        model: str | None = None,
+        on_step: Callable[[str, dict], None] | None = None,
+        read_only: bool = False,
+    ) -> None:
+        self._graph = graph
+        self._model = model or os.environ.get("ORCHESTRATOR_MODEL", os.environ.get("GRAPH_AGENT_MODEL", _DEFAULT_MODEL))
+        self._on_step = on_step
+        self._read_only = read_only
+        self._client = OpenAI(
+            api_key=os.environ["OPENAI_API_KEY"],
+            base_url=os.environ.get("OPENAI_API_BASE"),
+        )
+        self._history: list[dict] = [{"role": "system", "content": _SYSTEM_PROMPT}]
+
+    # ------------------------------------------------------------------
+    # Public
+    # ------------------------------------------------------------------
+
+    def chat(self, user_message: str) -> str:
+        """Route a message and return the combined agent response."""
+        self._history.append({"role": "user", "content": user_message})
+        result = self._run_loop()
+        self._history.append({"role": "assistant", "content": result})
+        return result
+
+    def reset(self) -> None:
+        self._history = [self._history[0]]
+
+    # ------------------------------------------------------------------
+    # Internal loop
+    # ------------------------------------------------------------------
+
+    def _run_loop(self) -> str:
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "run_graph_agent",
+                    "description": (
+                        "Delegate a task to the GraphAgent. Use for adding, updating, "
+                        "searching, linking, or enriching knowledge nodes."
+                    ),
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "message": {
+                                "type": "string",
+                                "description": "Full task description for the graph agent",
+                            },
+                        },
+                        "required": ["message"],
+                    },
+                },
+            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "run_review_agent",
+                    "description": (
+                        "Delegate a quality-review session to the ReviewAgent. "
+                        "Use for auditing, cleaning, deduplicating, or fixing existing nodes."
+                    ),
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "instructions": {
+                                "type": "string",
+                                "description": "Specific review focus (leave empty for general review)",
+                                "default": "",
+                            },
+                            "batch_size": {
+                                "type": "integer",
+                                "description": "Number of nodes to review in this session",
+                                "default": 5,
+                            },
+                        },
+                    },
+                },
+            },
+        ]
+
+        MAX_ITERATIONS = 8
+        for i in range(MAX_ITERATIONS):
+            if self._on_step:
+                self._on_step("orchestrator_thinking", {"iteration": i + 1})
+
+            response = self._client.chat.completions.create(
+                model=self._model,
+                messages=self._history,
+                tools=tools,
+                tool_choice="auto",
+            )
+            message = response.choices[0].message
+
+            if not message.tool_calls:
+                return message.content or ""
+
+            self._history.append(message.model_dump(exclude_unset=True))
+
+            for tc in message.tool_calls:
+                name = tc.function.name
+                try:
+                    kwargs = json.loads(tc.function.arguments or "{}")
+                except json.JSONDecodeError:
+                    kwargs = {}
+
+                if self._on_step:
+                    self._on_step("route", {"agent": name, "args": kwargs})
+
+                result = self._dispatch(name, kwargs)
+
+                self._history.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tc.id,
+                        "content": json.dumps({"result": result}, default=str),
+                    }
+                )
+
+        return "Orchestrator reached maximum iterations."
+
+    def _dispatch(self, name: str, kwargs: dict) -> str:
+        if name == "run_graph_agent":
+            return self._run_graph_agent(kwargs.get("message", ""))
+        if name == "run_review_agent":
+            return self._run_review_agent(
+                kwargs.get("instructions", ""),
+                int(kwargs.get("batch_size", 5)),
+            )
+        return f"Unknown agent: {name}"
+
+    def _run_graph_agent(self, message: str) -> str:
+        from agents.graph_agent.agent import GraphAgent
+
+        agent = GraphAgent(graph=self._graph, model=self._model, on_step=self._on_step, read_only=self._read_only)
+        return agent.chat(message)
+
+    def _run_review_agent(self, instructions: str, batch_size: int) -> str:
+        from agents.review_agent.agent import ReviewAgent
+
+        agent = ReviewAgent(
+            graph=self._graph,
+            model=self._model,
+            batch_size=batch_size,
+            on_step=self._on_step,
+        )
+        return agent.run_review(instructions=instructions)

agents/review_agent/agent.py

@@ -0,0 +1,188 @@
+"""ReviewAgent — LLM-driven agent that incrementally audits and cleans the graph.
+
+The agent never receives the full graph at once. Instead it:
+1. Checks overall graph health via ``get_graph_summary``.
+2. Samples a small batch of under-reviewed nodes via ``sample_nodes_for_review``.
+3. Inspects each node and its local neighbourhood with ``inspect_node``.
+4. Applies targeted fixes: normalise titles, move acronyms to aliases,
+   fix tags, merge near-duplicates, add missing edges.
+5. Marks each reviewed node with ``mark_reviewed`` so the next session
+   continues where this one left off.
+
+Configuration:
+    OPENAI_API_KEY     — required
+    OPENAI_API_BASE    — optional base URL override
+    REVIEW_AGENT_MODEL — model name (defaults to GRAPH_AGENT_MODEL or "qwen-plus")
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Callable
+
+from openai import OpenAI
+
+from core.graph.graph import KnowDoGraph
+from agents.review_agent.tools import REVIEW_TOOL_DISPATCH, REVIEW_TOOL_SCHEMAS
+
+_DEFAULT_MODEL = "qwen-plus"
+
+_SYSTEM_PROMPT = """You are a knowledge-graph quality reviewer for the Know-Do Graph system.
+
+Your role is to incrementally audit existing nodes (entries) for quality issues and
+fix them. You work in focused batches — you do NOT need to review the entire graph
+in one session.
+
+## Quality criteria
+1. **Title hygiene**
+   - Titles must be concise (3–7 words).
+   - Remove parenthetical acronyms/aliases from titles — put them in `aliases` instead.
+     Example: "Density Functional Theory (DFT)" → title "Density Functional Theory",
+     aliases ["DFT", "ab initio DFT"].
+   - Remove redundant tool-name prefixes from capability titles.
+     Example: "RDKit Molecular Fingerprint Generation" → "Molecular Fingerprint Generation"
+     (keep "rdkit" as a tag and ensure a `dependency` edge to the RDKit tool node).
+
+2. **Tag normalisation**
+   - All tags must be lowercase and hyphenated (e.g. "machine-learning", "rdkit").
+   - Remove capitalised duplicates (e.g. "RDKit" when "rdkit" already present).
+   - Tags should be domain-specific and meaningful — remove generic filler tags.
+
+3. **Duplicate / alias detection**
+   - If two nodes represent the same concept, merge them with `merge_entries`.
+   - If one node is a more specific variant of another, add a `derived_from` or
+     `refinement_of` edge rather than merging.
+
+4. **Edge completeness**
+   - If a node uses a tool or library, ensure a `dependency` edge exists to that
+     tool's node.
+   - If a node is a specific application of a broader capability, add a
+     `derived_from` or `prerequisite` edge.
+
+## Your workflow each session
+1. Call `get_graph_summary` to see overall health and review coverage.
+2. Call `sample_nodes_for_review` to get a batch (default 5).
+3. For each sampled node: call `inspect_node`, assess quality, apply fixes if needed.
+4. After inspecting (and optionally fixing) each node, call `mark_reviewed`.
+5. Summarise what you found and fixed.
+
+Keep changes conservative — prefer targeted fixes over large rewrites.
+"""
+
+
+class ReviewAgent:
+    """LLM-powered agent that reviews and cleans the Know-Do Graph incrementally.
+
+    Parameters
+    ----------
+    graph:
+        The shared ``KnowDoGraph`` instance.
+    model:
+        Model identifier forwarded to the OpenAI client.
+    batch_size:
+        Number of nodes to review per ``run_review`` call.
+    """
+
+    def __init__(
+        self,
+        graph: KnowDoGraph,
+        model: str | None = None,
+        batch_size: int = 5,
+        on_step: Callable[[str, dict], None] | None = None,
+    ) -> None:
+        self._graph = graph
+        self._batch_size = batch_size
+        self._on_step = on_step
+        self._model = model or os.environ.get(
+            "REVIEW_AGENT_MODEL",
+            os.environ.get("GRAPH_AGENT_MODEL", _DEFAULT_MODEL),
+        )
+        self._client = OpenAI(
+            api_key=os.environ["OPENAI_API_KEY"],
+            base_url=os.environ.get("OPENAI_API_BASE"),
+        )
+
+    # ------------------------------------------------------------------
+    # Public interface
+    # ------------------------------------------------------------------
+
+    def run_review(self, instructions: str = "") -> str:
+        """Run one review session and return a summary of findings and fixes."""
+        user_msg = (
+            f"Please review a batch of {self._batch_size} nodes. "
+            + (instructions if instructions else "Apply all quality criteria from your instructions.")
+        )
+        history: list[dict] = [
+            {"role": "system", "content": _SYSTEM_PROMPT},
+            {"role": "user", "content": user_msg},
+        ]
+        return self._run_loop(history)
+
+    def chat(self, user_message: str) -> str:
+        """Single-turn interactive review conversation (stateless)."""
+        history: list[dict] = [
+            {"role": "system", "content": _SYSTEM_PROMPT},
+            {"role": "user", "content": user_message},
+        ]
+        return self._run_loop(history)
+
+    # ------------------------------------------------------------------
+    # Internal agentic loop
+    # ------------------------------------------------------------------
+
+    def _run_loop(self, history: list[dict]) -> str:
+        MAX_ITERATIONS = 30
+        for i in range(MAX_ITERATIONS):
+            if self._on_step:
+                self._on_step("thinking", {"iteration": i + 1})
+
+            response = self._client.chat.completions.create(
+                model=self._model,
+                messages=history,
+                tools=REVIEW_TOOL_SCHEMAS,
+                tool_choice="auto",
+            )
+            message = response.choices[0].message
+
+            if not message.tool_calls:
+                return message.content or ""
+
+            history.append(message.model_dump(exclude_unset=True))
+
+            for tc in message.tool_calls:
+                try:
+                    display_args = {k: v for k, v in json.loads(tc.function.arguments or "{}").items() if k != "graph"}
+                except Exception:
+                    display_args = {}
+                if self._on_step:
+                    self._on_step("tool_call", {"name": tc.function.name, "args": display_args})
+
+                result = self._dispatch(tc.function.name, tc.function.arguments)
+
+                if self._on_step:
+                    self._on_step("tool_result", {"name": tc.function.name, "result": result})
+
+                history.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tc.id,
+                        "content": json.dumps(result, default=str),
+                    }
+                )
+
+        return "Review agent reached maximum iterations without a final answer."
+
+    def _dispatch(self, name: str, arguments_json: str) -> Any:
+        func = REVIEW_TOOL_DISPATCH.get(name)
+        if func is None:
+            return {"error": f"Unknown tool: {name}"}
+        try:
+            kwargs = json.loads(arguments_json) if arguments_json else {}
+        except json.JSONDecodeError as exc:
+            return {"error": f"Bad arguments JSON: {exc}"}
+        kwargs["graph"] = self._graph
+        try:
+            return func(**kwargs)
+        except Exception as exc:  # noqa: BLE001
+            return {"error": str(exc)}