know-do-graph 0.1.0__py3-none-any.whl

api/routes/remote_sync.py

@@ -0,0 +1,230 @@
+"""Remote-source sync routes.
+
+These endpoints expose the :mod:`core.sync.remote_sync` machinery so that
+operators (and the UI) can:
+
+  * list all entries that mirror an upstream file,
+  * trigger a one-shot resync (one entry or all due),
+  * attach / detach a remote source on an existing entry.
+
+Mounted at ``/remote-sync`` from :mod:`api.main`.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from core import events as _events
+from core.app_state import graph as _graph
+from core.schemas.entry import RemoteSource
+from core.storage.database import get_db
+from core.storage.repository import EntryRepository
+from core.sync.remote_sync import (
+    SyncResult,
+    parse_github_url,
+    sync_all_due,
+    sync_entry,
+)
+
+router = APIRouter()
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _result_to_dict(r: SyncResult) -> dict:
+    return {
+        "entry_id": r.entry_id,
+        "title": r.title,
+        "status": r.status,
+        "detail": r.detail,
+        "bytes_fetched": r.bytes_fetched,
+        "new_hash": r.new_hash,
+        "fetched_at": r.fetched_at.isoformat() if r.fetched_at else None,
+    }
+
+
+def _source_to_dict(src: RemoteSource) -> dict:
+    d = src.model_dump(mode="json")
+    if isinstance(d.get("fetched_at"), datetime):
+        d["fetched_at"] = d["fetched_at"].isoformat()
+    return d
+
+
+def _resolve_entry(db: Session, id_or_slug: str):
+    """Look up an entry by id, slug, or alias via EntryRepository.get_all().
+
+    Cheap because get_all() is O(n) over a small node set; if this gets hot,
+    add an explicit get_by_slug method on the repo.
+    """
+    repo = EntryRepository(db)
+    for e in repo.get_all():
+        if e.id == id_or_slug or e.slug == id_or_slug or id_or_slug in e.aliases:
+            return repo, e
+    raise HTTPException(status_code=404, detail=f"entry not found: {id_or_slug}")
+
+
+# ── Request models ────────────────────────────────────────────────────────────
+
+
+class AttachSourceRequest(BaseModel):
+    url: str
+    ref: Optional[str] = None
+    path: Optional[str] = None
+    owner: Optional[str] = None
+    repo: Optional[str] = None
+    auto_sync: bool = True
+    sync_interval_seconds: int = 3600
+    sync_now: bool = True
+
+
+# ── Routes ────────────────────────────────────────────────────────────────────
+
+
+@router.get("/")
+def list_linked_entries(db: Session = Depends(get_db)) -> list[dict]:
+    """List every entry that mirrors an upstream source."""
+    out: list[dict] = []
+    for e in EntryRepository(db).get_all():
+        src = e.metadata.remote_source
+        if src is None:
+            continue
+        out.append({
+            "entry_id": e.id,
+            "slug": e.slug,
+            "title": e.title,
+            "remote_source": _source_to_dict(src),
+        })
+    return out
+
+
+@router.post("/all")
+async def sync_all_endpoint(force: bool = False) -> dict:
+    """Sync every entry whose remote source is due (or all when ``force=true``)."""
+    results = await sync_all_due(force=force)
+    summary = {
+        "checked": len(results),
+        "updated": sum(1 for r in results if r.status == "updated"),
+        "unchanged": sum(1 for r in results if r.status == "unchanged"),
+        "errors": sum(1 for r in results if r.status == "error"),
+        "results": [_result_to_dict(r) for r in results],
+    }
+    return summary
+
+
+@router.post("/{id_or_slug}")
+async def sync_one_endpoint(
+    id_or_slug: str,
+    force: bool = True,
+    db: Session = Depends(get_db),
+) -> dict:
+    """Sync a single entry now."""
+    repo, entry = _resolve_entry(db, id_or_slug)
+    if entry.metadata.remote_source is None:
+        raise HTTPException(status_code=400, detail="entry has no remote_source")
+    result = await sync_entry(entry, force=force)
+    updated = repo.update(entry)
+    autolink_summary: dict | None = None
+    if result.status == "updated" and updated is not None:
+        try:
+            _graph.add_entry(updated)
+        except Exception:
+            pass
+        # Refresh derived edges from the new content.
+        try:
+            from core.storage.repository import EdgeRepository
+            from core.sync.autolink import auto_link_entry
+            al = auto_link_entry(updated, repo.get_all(), EdgeRepository(db))
+            autolink_summary = {
+                "frontmatter_edges": al.frontmatter_edges,
+                "mention_edges": al.mention_edges,
+            }
+        except Exception:  # pragma: no cover
+            pass
+        _events.emit(
+            "node_updated",
+            {"id": entry.id, "slug": entry.slug, "title": entry.title, "source": "remote_sync"},
+        )
+    return {
+        "result": _result_to_dict(result),
+        "remote_source": _source_to_dict(entry.metadata.remote_source),
+        "autolink": autolink_summary,
+    }
+
+
+@router.put("/{id_or_slug}/source")
+async def attach_source(
+    id_or_slug: str,
+    body: AttachSourceRequest,
+    db: Session = Depends(get_db),
+) -> dict:
+    """Attach or replace the remote source on an entry."""
+    repo, entry = _resolve_entry(db, id_or_slug)
+
+    owner = body.owner
+    repo_name = body.repo
+    path = body.path
+    ref = body.ref
+    kind = "github"
+
+    parsed = parse_github_url(body.url)
+    if parsed:
+        owner = owner or parsed.get("owner")
+        repo_name = repo_name or parsed.get("repo")
+        path = path or parsed.get("path")
+        ref = ref or parsed.get("ref") or "main"
+    else:
+        kind = "http"
+
+    if kind == "github" and not (owner and repo_name and path):
+        raise HTTPException(
+            status_code=400,
+            detail="GitHub source needs owner/repo/path (parsed from URL or supplied explicitly)",
+        )
+
+    src = RemoteSource(
+        kind=kind,
+        url=body.url,
+        owner=owner,
+        repo=repo_name,
+        ref=ref or "main",
+        path=path,
+        auto_sync=body.auto_sync,
+        sync_interval_seconds=body.sync_interval_seconds,
+    )
+    entry.metadata.remote_source = src
+    repo.update(entry)
+
+    result_dict = None
+    if body.sync_now:
+        result = await sync_entry(entry, force=True)
+        updated = repo.update(entry)
+        if result.status == "updated" and updated is not None:
+            try:
+                _graph.add_entry(updated)
+            except Exception:
+                pass
+            _events.emit(
+                "node_updated",
+                {"id": entry.id, "slug": entry.slug, "title": entry.title, "source": "remote_sync"},
+            )
+        result_dict = _result_to_dict(result)
+
+    return {
+        "remote_source": _source_to_dict(entry.metadata.remote_source),
+        "result": result_dict,
+    }
+
+
+@router.delete("/{id_or_slug}/source")
+def detach_source(id_or_slug: str, db: Session = Depends(get_db)) -> dict:
+    """Remove the remote source link from an entry (content is preserved)."""
+    repo, entry = _resolve_entry(db, id_or_slug)
+    entry.metadata.remote_source = None
+    repo.update(entry)
+    return {"detached": True, "entry_id": entry.id}

api/routes/retrieve.py

@@ -0,0 +1,88 @@
+"""Progressive (staged) retrieval API.
+
+Endpoints surface the hierarchical-memory layers so external agents can pull
+only the level of detail needed for the current execution stage.
+
+See :mod:`core.retrieval.progressive` for the underlying logic.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from core.app_state import graph as _graph
+from core.retrieval.progressive import ProgressiveRetriever
+from core.schemas.entry import Entry, implied_level
+from core.storage.database import get_db
+
+router = APIRouter()
+
+
+def _retriever(db: Session = Depends(get_db)) -> ProgressiveRetriever:
+    return ProgressiveRetriever(db, _graph)
+
+
+def _annotate(entry: Entry) -> dict:
+    data = entry.model_dump(mode="json")
+    level = implied_level(entry.entry_type, entry.metadata.skill_level)
+    data["_level"] = level.value if level else None
+    return data
+
+
+@router.get("/plan", response_model=list[dict])
+def plan(
+    goal: str = Query(..., description="Free-text description of what you want to do"),
+    k: int = Query(5, ge=1, le=50),
+    mode: str = Query("hybrid", pattern="^(hybrid|semantic|keyword)$"),
+    include_l2: bool = Query(True, description="Include L2 procedures alongside L1 capabilities"),
+    retriever: ProgressiveRetriever = Depends(_retriever),
+):
+    """Return planner-level candidates (L1 capabilities, optionally L2 procedures).
+
+    Heuristics (L3) and constraints (L4) are intentionally excluded — fetch
+    them on demand via ``/retrieve/heuristics`` and ``/retrieve/constraints``.
+    """
+    return [_annotate(e) for e in retriever.plan(goal=goal, k=k, mode=mode, include_l2=include_l2)]
+
+
+@router.get("/heuristics", response_model=list[dict])
+def heuristics(
+    skill: str = Query(..., description="Entry id, slug, or alias of the L1/L2 skill"),
+    k: int = Query(5, ge=1, le=50),
+    fallback: bool = Query(True, description="Include semantic-search L3 fallback if no edges exist"),
+    retriever: ProgressiveRetriever = Depends(_retriever),
+):
+    """Return L3 heuristics attached to a skill."""
+    return [_annotate(e) for e in retriever.heuristics_for(skill, k=k, include_semantic_fallback=fallback)]
+
+
+@router.get("/constraints", response_model=list[dict])
+def constraints(
+    skill: str = Query(..., description="Entry id, slug, or alias of the L1/L2 skill"),
+    k: int = Query(5, ge=1, le=50),
+    fallback: bool = Query(True, description="Include semantic-search L4 fallback if no edges exist"),
+    retriever: ProgressiveRetriever = Depends(_retriever),
+):
+    """Return L4 constraints / failure modes attached to a skill."""
+    return [_annotate(e) for e in retriever.constraints_for(skill, k=k, include_semantic_fallback=fallback)]
+
+
+@router.get("/expand/{skill}", response_model=dict)
+def expand(
+    skill: str,
+    stages: Optional[str] = Query(
+        None,
+        description="Comma-separated subset of heuristics,constraints,decomposition (default: heuristics,constraints)",
+    ),
+    k: int = Query(5, ge=1, le=50),
+    retriever: ProgressiveRetriever = Depends(_retriever),
+):
+    """Bundle additional context for an already-selected skill (verifier loop)."""
+    stage_list = [s.strip() for s in stages.split(",")] if stages else None
+    result = retriever.expand(skill=skill, stages=stage_list, k=k)
+    if "error" in result:
+        raise HTTPException(status_code=404, detail=result["error"])
+    return result

core/app_state.py

@@ -0,0 +1,9 @@
+"""Application-level shared state.
+
+Import `graph` from here wherever a single in-process graph instance is needed.
+The API startup handler calls `graph.rebuild_from_db(...)` after init_db().
+"""
+
+from core.graph.graph import KnowDoGraph
+
+graph = KnowDoGraph()

core/events.py

@@ -0,0 +1,84 @@
+"""Lightweight SSE event bus for broadcasting graph mutations to connected clients.
+
+Usage (from sync FastAPI route handlers):
+    from core import events
+    events.emit("node_added", {"id": ..., "title": ...})
+
+Usage (SSE endpoint):
+    q = events.subscribe()
+    try:
+        msg = await asyncio.wait_for(q.get(), timeout=25)
+        if msg is events.SHUTDOWN_SENTINEL:
+            return  # server shutting down
+    finally:
+        events.unsubscribe(q)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+
+# Sentinel put into every subscriber queue on server shutdown
+SHUTDOWN_SENTINEL: object = object()
+
+# Active subscriber queues (one per SSE connection)
+_subscribers: set[asyncio.Queue] = set()
+
+# The asyncio loop FastAPI runs on. Captured during app startup so that
+# `emit()` works when invoked from worker threads (sync route handlers and
+# background threads have no `get_running_loop()`).
+_loop: asyncio.AbstractEventLoop | None = None
+
+
+def set_loop(loop: asyncio.AbstractEventLoop) -> None:
+    """Remember the main event loop so `emit()` can dispatch from any thread."""
+    global _loop
+    _loop = loop
+
+
+def signal_shutdown() -> None:
+    """Push the shutdown sentinel into every subscriber queue so generators exit cleanly."""
+    for q in list(_subscribers):
+        try:
+            q.put_nowait(SHUTDOWN_SENTINEL)
+        except asyncio.QueueFull:
+            pass
+
+
+def subscribe() -> asyncio.Queue:
+    """Register a new SSE subscriber and return its queue."""
+    q: asyncio.Queue = asyncio.Queue(maxsize=200)
+    _subscribers.add(q)
+    return q
+
+
+def unsubscribe(q: asyncio.Queue) -> None:
+    """Remove a subscriber queue (called when the SSE connection closes)."""
+    _subscribers.discard(q)
+
+
+def emit(event_type: str, data: Any = None) -> None:
+    """Broadcast a graph-change event to all connected SSE clients.
+
+    Safe to call from sync FastAPI route handlers (which run in a thread pool)
+    and from background threads — delivery is scheduled on the main asyncio
+    event loop captured at startup via :func:`set_loop`.
+    """
+    msg = json.dumps({"type": event_type, "data": data or {}})
+    loop = _loop
+    if loop is None:
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return  # No event loop available — nothing to broadcast
+    for q in list(_subscribers):
+        loop.call_soon_threadsafe(_try_put, q, msg)
+
+
+def _try_put(q: asyncio.Queue, msg: str) -> None:
+    try:
+        q.put_nowait(msg)
+    except asyncio.QueueFull:
+        pass  # Slow consumer — drop the event rather than blocking

core/extraction/wikilink_parser.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import re
+
+_WIKILINK_RE = re.compile(r"\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]")
+_MDLINK_RE = re.compile(r"\[(?:[^\]]+)\]\((https?://[^\)]+)\)")
+
+
+def parse_wikilinks(content: str) -> list[str]:
+    """Return all [[wikilink]] targets found in *content*."""
+    return _WIKILINK_RE.findall(content)
+
+
+def extract_external_refs(content: str) -> list[str]:
+    """Return all markdown [text](url) hrefs found in *content*."""
+    return _MDLINK_RE.findall(content)
+
+
+_CHAR_SUBS: dict[str, str] = {
+    "Å": "angstrom",
+    "å": "angstrom",
+    "µ": "micro",
+    "μ": "micro",
+    "°": "deg",
+    "±": "plus-minus",
+    "×": "x",
+    "·": "-",
+}
+
+
+def slug_from_title(title: str) -> str:
+    import unicodedata
+    for sym, replacement in _CHAR_SUBS.items():
+        title = title.replace(sym, f" {replacement} ")
+    parts: list[str] = []
+    for ch in unicodedata.normalize("NFKD", title):
+        if ch.isascii():
+            parts.append(ch)
+        elif unicodedata.combining(ch):
+            pass
+        else:
+            name = unicodedata.name(ch, "").lower()
+            parts.append(name.split()[-1] if name else "")
+    slug = "".join(parts).lower().strip()
+    slug = re.sub(r"[^\w\s-]", "", slug)
+    slug = re.sub(r"[\s_]+", "-", slug)
+    slug = re.sub(r"-+", "-", slug)
+    return slug.strip("-")

core/graph/graph.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+import networkx as nx
+
+from core.schemas.edge import Edge, EdgeRelation
+from core.schemas.entry import Entry
+
+logger = logging.getLogger(__name__)
+
+
+class KnowDoGraph:
+    """In-memory directed graph backed by networkx.
+
+    Entries are nodes; edges represent semantic relations between them.
+    This is rebuilt from the database on startup and kept in sync during
+    the process lifetime.
+    """
+
+    def __init__(self) -> None:
+        self._g: nx.DiGraph = nx.DiGraph()
+
+    # ------------------------------------------------------------------
+    # Nodes
+    # ------------------------------------------------------------------
+
+    def add_entry(self, entry: Entry) -> None:
+        md = entry.metadata
+        timestamp = md.timestamp.isoformat() if getattr(md, "timestamp", None) else None
+        verification = (
+            md.verification_status.value
+            if hasattr(md.verification_status, "value")
+            else md.verification_status
+        )
+        # Effective hierarchical-memory level (explicit override > entry_type default).
+        from core.schemas.entry import implied_level
+
+        level_obj = implied_level(entry.entry_type, md.skill_level)
+        level_value = level_obj.value if level_obj else None
+        self._g.add_node(
+            entry.id,
+            title=entry.title,
+            slug=entry.slug,
+            entry_type=entry.entry_type.value,
+            tags=entry.tags,
+            timestamp=timestamp,
+            usage_count=md.usage_count,
+            trust_score=md.trust_score,
+            verification_status=verification,
+            skill_level=level_value,
+        )
+
+    def remove_entry(self, entry_id: str) -> None:
+        if self._g.has_node(entry_id):
+            self._g.remove_node(entry_id)
+
+    # ------------------------------------------------------------------
+    # Edges
+    # ------------------------------------------------------------------
+
+    def add_edge(self, edge: Edge) -> bool:
+        """Add an edge to the in-memory graph.
+
+        Returns ``False`` (and logs a warning) if either endpoint is unknown,
+        instead of silently letting networkx auto-create a typeless ghost node.
+        """
+        if not self._g.has_node(edge.source_id) or not self._g.has_node(edge.target_id):
+            logger.warning(
+                "skipping edge %s → %s (%s): endpoint missing from graph",
+                edge.source_id,
+                edge.target_id,
+                edge.relation.value if hasattr(edge.relation, "value") else edge.relation,
+            )
+            return False
+        self._g.add_edge(
+            edge.source_id,
+            edge.target_id,
+            id=edge.id,
+            relation=edge.relation.value if hasattr(edge.relation, "value") else edge.relation,
+            weight=edge.weight,
+        )
+        return True
+
+    def remove_edge(self, source_id: str, target_id: str) -> None:
+        if self._g.has_edge(source_id, target_id):
+            self._g.remove_edge(source_id, target_id)
+
+    # ------------------------------------------------------------------
+    # Queries
+    # ------------------------------------------------------------------
+
+    def get_neighbors(
+        self,
+        entry_id: str,
+        relation: Optional[EdgeRelation] = None,
+        direction: str = "both",
+    ) -> list[dict]:
+        """Return neighboring node IDs with edge metadata.
+
+        direction: "out" (successors), "in" (predecessors), "both"
+        """
+        if entry_id not in self._g:
+            return []
+
+        neighbors: list[dict] = []
+
+        def _matches(data: dict) -> bool:
+            if relation is None:
+                return True
+            rel_val = relation.value if hasattr(relation, "value") else relation
+            return data.get("relation") == rel_val
+
+        if direction in ("out", "both"):
+            for nbr in self._g.successors(entry_id):
+                data = dict(self._g.edges[entry_id, nbr])
+                if _matches(data):
+                    neighbors.append({"id": nbr, "direction": "out", **data})
+
+        if direction in ("in", "both"):
+            for nbr in self._g.predecessors(entry_id):
+                data = dict(self._g.edges[nbr, entry_id])
+                if _matches(data):
+                    neighbors.append({"id": nbr, "direction": "in", **data})
+
+        return neighbors
+
+    def get_related_ids(
+        self,
+        entry_id: str,
+        depth: int = 1,
+        relation: Optional[EdgeRelation] = None,
+    ) -> list[str]:
+        """BFS from *entry_id* up to *depth* hops, optionally filtered by relation type.
+
+        Returns IDs of all reachable nodes (excluding the start node).
+        """
+        visited: set[str] = {entry_id}
+        frontier: set[str] = {entry_id}
+        for _ in range(depth):
+            next_frontier: set[str] = set()
+            for node in frontier:
+                for nbr_info in self.get_neighbors(node, relation=relation):
+                    nbr_id = nbr_info["id"]
+                    if nbr_id not in visited:
+                        next_frontier.add(nbr_id)
+            frontier = next_frontier
+            visited.update(frontier)
+        visited.discard(entry_id)
+        return list(visited)
+
+    def has_node(self, entry_id: str) -> bool:
+        return self._g.has_node(entry_id)
+
+    def get_subgraph(self, entry_id: str, depth: int = 2) -> nx.DiGraph:
+        """Return an ego-subgraph centred on entry_id up to *depth* hops."""
+        if entry_id not in self._g:
+            return nx.DiGraph()
+        nodes = {entry_id}
+        frontier = {entry_id}
+        for _ in range(depth):
+            next_frontier: set[str] = set()
+            for node in frontier:
+                next_frontier.update(self._g.successors(node))
+                next_frontier.update(self._g.predecessors(node))
+            frontier = next_frontier - nodes
+            nodes.update(frontier)
+        return self._g.subgraph(nodes).copy()
+
+    def find_paths(
+        self, source_id: str, target_id: str, cutoff: int = 6
+    ) -> list[list[str]]:
+        try:
+            return list(
+                nx.all_simple_paths(self._g, source_id, target_id, cutoff=cutoff)
+            )
+        except (nx.NodeNotFound, nx.NetworkXNoPath):
+            return []
+
+    def stats(self) -> dict:
+        return {
+            "nodes": self._g.number_of_nodes(),
+            "edges": self._g.number_of_edges(),
+            "is_dag": nx.is_directed_acyclic_graph(self._g),
+        }
+
+    def rebuild_from_db(self, entries: list[Entry], edges: list[Edge]) -> None:
+        """Clear and rebuild the graph from persisted entries and edges.
+
+        Edges whose endpoints are not present in *entries* are skipped (with a
+        warning). They survive in the database — the maintenance agent's
+        ``remove_dangling_edges`` is responsible for pruning them — but they
+        are never allowed to materialise ghost nodes in the in-memory graph.
+        """
+        self._g.clear()
+        for entry in entries:
+            self.add_entry(entry)
+        skipped = 0
+        for edge in edges:
+            if not self.add_edge(edge):
+                skipped += 1
+        if skipped:
+            logger.warning("rebuild_from_db: skipped %d dangling edge(s)", skipped)