pulse8-ai-cortex-knowledge-vault 0.3.0__py3-none-any.whl

cortex/__init__.py

@@ -0,0 +1 @@
+"""Cortex — Agent-native knowledge OS built on Markdown files."""

cortex/api/routes.py

@@ -0,0 +1,328 @@
+"""REST API route handlers for the Cortex vault."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from cortex.graph.engine import GraphEngine
+from cortex.log.audit import log_operation
+from cortex.search.qmd import QMDSearch
+from cortex.vault.index import rebuild_index
+from cortex.vault.models import Edge, EdgeType
+from cortex.vault.reader import read_note, resolve_wikilink, scan_vault
+from cortex.vault.writer import write_note
+
+router = APIRouter()
+
+
+class WriteNoteBody(BaseModel):
+    """Request body for creating or updating a note."""
+    content: str
+    frontmatter: Optional[dict[str, Any]] = None
+    mode: str = "upsert"
+    authored_by: str = "human"
+    model: Optional[str] = None
+
+
+class CreateLinkBody(BaseModel):
+    """Request body for creating a graph edge."""
+    source: str
+    target: str
+    edge_type: str
+    metadata: Optional[dict[str, Any]] = None
+
+
+class IngestBody(BaseModel):
+    """Request body for ingesting a raw source."""
+    content: str
+    filename: str
+    source_type: str = "text"
+    auto_compile: bool = False
+
+
+def get_vault_path(request: Request):
+    """Extract the vault path from application state."""
+    return request.app.state.vault_path
+
+
+def get_graph(request: Request) -> GraphEngine:
+    """Extract the graph engine from application state."""
+    return request.app.state.graph
+
+
+def get_qmd(request: Request) -> QMDSearch:
+    """Extract the QMD search backend from application state."""
+    return request.app.state.qmd
+
+
+def get_qmd_debounce(request: Request):
+    """Extract the debounced QMD updater from application state."""
+    return request.app.state.qmd_debounce
+
+
+@router.get("/health")
+async def health():
+    """Liveness probe."""
+    return {"status": "healthy"}
+
+
+@router.get("/notes/{path:path}")
+async def read_note_endpoint(path: str, request: Request):
+    """Read a note by vault-relative path."""
+    vault_path = get_vault_path(request)
+    graph = get_graph(request)
+
+    try:
+        note = read_note(vault_path / path, vault_path)
+    except FileNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=f"Note not found: {path}") from exc
+
+    edges = await graph.get_edges(note.path)
+    edge_dicts = [
+        {"source": e.source, "target": e.target, "edge_type": e.edge_type.value}
+        for e in edges
+    ]
+
+    await log_operation(vault_path, "api", "vault:read", f"Read {path}")
+
+    return {
+        "path": note.path,
+        "title": note.title,
+        "content": note.content,
+        "frontmatter": note.frontmatter,
+        "node_type": note.node_type.value,
+        "wikilinks": note.wikilinks,
+        "tags": note.tags,
+        "edges": edge_dicts,
+    }
+
+
+@router.put("/notes/{path:path}")
+async def write_note_endpoint(path: str, body: WriteNoteBody, request: Request):
+    """Create or update a note with provenance tracking."""
+    vault_path = get_vault_path(request)
+    graph = get_graph(request)
+    qmd_debounce = get_qmd_debounce(request)
+
+    try:
+        note = write_note(
+            path=vault_path / path,
+            vault_root=vault_path,
+            content=body.content,
+            frontmatter=body.frontmatter,
+            mode=body.mode,
+            authored_by=body.authored_by,
+            model=body.model,
+        )
+
+        await graph.add_note_node(note)
+
+        for tag in note.tags:
+            tag_id = f"tag:{tag}"
+            if not graph.graph.has_node(tag_id):
+                graph.graph.add_node(tag_id, node_type="tag", title=tag)
+            await graph.add_edge(
+                Edge(source=note.path, target=tag_id, edge_type=EdgeType.TAGGED_WITH)
+            )
+
+        for link in note.wikilinks:
+            resolved = resolve_wikilink(link, vault_path)
+            if resolved:
+                await graph.add_edge(
+                    Edge(source=note.path, target=resolved, edge_type=EdgeType.LINKS_TO)
+                )
+
+        await rebuild_index(vault_path)
+        qmd_debounce.schedule()
+        await log_operation(vault_path, body.authored_by, "vault:write", f"Wrote {path}")
+
+        return {"path": note.path, "title": note.title, "status": "written"}
+
+    except FileExistsError as exc:
+        raise HTTPException(status_code=409, detail=f"Note already exists: {path}") from exc
+    except FileNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=f"Note not found: {path}") from exc
+
+
+@router.get("/search")
+async def search_endpoint(
+    q: str,
+    request: Request,
+    mode: str | None = None,
+    collection: Optional[str] = None,
+    top_k: int = 10,
+):
+    """Search the vault via QMD and return graph-enriched results."""
+    from cortex.config import settings
+
+    vault_path = get_vault_path(request)
+    graph = get_graph(request)
+    qmd = get_qmd(request)
+    effective_mode = mode or settings.qmd_search_mode
+
+    raw_results = await qmd.search(q, mode=effective_mode, collection=collection, top_k=top_k)
+
+    paths = [r.get("path", "") for r in raw_results]
+    edges_by_path = await graph.get_edges_batch(paths)
+
+    enriched = []
+    for r in raw_results:
+        path = r.get("path", "")
+        edges = edges_by_path.get(path, [])
+        edge_dicts = [
+            {"source": e.source, "target": e.target, "edge_type": e.edge_type.value}
+            for e in edges
+        ]
+        enriched.append({**r, "edges": edge_dicts})
+
+    await log_operation(vault_path, "api", "vault:search", f"Search: {q}")
+    return {"query": q, "mode": mode, "results": enriched}
+
+
+@router.post("/links")
+async def create_link_endpoint(body: CreateLinkBody, request: Request):
+    """Create a typed edge in the knowledge graph."""
+    vault_path = get_vault_path(request)
+    graph = get_graph(request)
+
+    edge = Edge(
+        source=body.source,
+        target=body.target,
+        edge_type=EdgeType(body.edge_type),
+        metadata=body.metadata or {},
+    )
+    await graph.add_edge(edge)
+    await log_operation(
+        vault_path, "api", "vault:link",
+        f"Created {body.edge_type}: {body.source} → {body.target}",
+    )
+    return {"status": "created", "source": body.source, "target": body.target, "edge_type": body.edge_type}
+
+
+@router.get("/links")
+async def query_links_endpoint(
+    source: str,
+    request: Request,
+    edge_type: Optional[str] = None,
+):
+    """Query edges from a source node."""
+    graph = get_graph(request)
+    edge_types = [EdgeType(edge_type)] if edge_type else None
+    edges = await graph.get_edges(source, edge_types=edge_types)
+    return {
+        "source": source,
+        "edges": [
+            {
+                "source": e.source,
+                "target": e.target,
+                "edge_type": e.edge_type.value,
+                "metadata": e.metadata,
+            }
+            for e in edges
+        ],
+    }
+
+
+@router.delete("/links/{source:path}")
+async def delete_link_endpoint(
+    source: str,
+    request: Request,
+    target: str = "",
+    edge_type: str = "",
+):
+    """Delete a typed edge from the knowledge graph."""
+    vault_path = get_vault_path(request)
+    graph = get_graph(request)
+
+    if not target or not edge_type:
+        raise HTTPException(status_code=400, detail="target and edge_type query params required")
+
+    await graph.remove_edge(source, target, EdgeType(edge_type))
+    await log_operation(
+        vault_path, "api", "vault:link",
+        f"Deleted {edge_type}: {source} → {target}",
+    )
+    return {"status": "deleted", "source": source, "target": target, "edge_type": edge_type}
+
+
+@router.get("/graph/stats")
+async def graph_stats_endpoint(request: Request):
+    """Return graph node/edge statistics."""
+    graph = get_graph(request)
+    return await graph.get_stats()
+
+
+@router.post("/ingest")
+async def ingest_endpoint(body: IngestBody, request: Request):
+    """Ingest a raw source file and optionally compile it."""
+    vault_path = get_vault_path(request)
+    qmd_debounce = get_qmd_debounce(request)
+
+    raw_path = vault_path / "raw" / body.filename
+    raw_path.parent.mkdir(parents=True, exist_ok=True)
+    raw_path.write_text(body.content)
+
+    rel_path = f"raw/{body.filename}"
+    await log_operation(vault_path, "api", "vault:ingest", f"Ingested {rel_path}")
+
+    result: dict[str, Any] = {
+        "path": rel_path,
+        "status": "ingested",
+        "compiled": False,
+    }
+
+    if body.auto_compile:
+        from cortex.compiler.compiler import KnowledgeCompiler
+
+        compiler = KnowledgeCompiler(vault_path)
+        created = await compiler.ingest_source(raw_path)
+        result["compiled"] = True
+        result["wiki_articles"] = [str(p.relative_to(vault_path)) for p in created]
+
+    qmd_debounce.schedule()
+
+    return result
+
+
+@router.post("/compile")
+async def compile_endpoint(request: Request):
+    """Compile all unprocessed raw sources into wiki articles."""
+    vault_path = get_vault_path(request)
+    qmd_debounce = get_qmd_debounce(request)
+
+    existing_sources: set[str] = set()
+    for note in scan_vault(vault_path):
+        sp = note.frontmatter.get("source_path")
+        if sp:
+            existing_sources.add(sp)
+
+    raw_dir = vault_path / "raw"
+    if not raw_dir.exists():
+        return {"status": "no raw directory", "compiled": 0}
+
+    from cortex.compiler.compiler import KnowledgeCompiler
+
+    compiler = KnowledgeCompiler(vault_path)
+    compiled_count = 0
+    all_created: list[str] = []
+
+    for raw_file in sorted(raw_dir.iterdir()):
+        if raw_file.is_dir():
+            continue
+        rel = str(raw_file.relative_to(vault_path))
+        if rel not in existing_sources:
+            created = await compiler.ingest_source(raw_file)
+            compiled_count += 1
+            all_created.extend(str(p.relative_to(vault_path)) for p in created)
+
+    await rebuild_index(vault_path)
+    qmd_debounce.schedule()
+    await log_operation(vault_path, "api", "vault:compile", f"Compiled {compiled_count} sources")
+
+    return {
+        "status": "compiled",
+        "sources_compiled": compiled_count,
+        "articles_created": all_created,
+    }

cortex/compiler/compiler.py

@@ -0,0 +1,151 @@
+"""LLM-powered knowledge compiler for raw source ingestion."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from openai import AsyncOpenAI
+
+from cortex.compiler.prompts import COMPILE_SYSTEM_PROMPT, INGEST_SYSTEM_PROMPT
+from cortex.config import settings
+from cortex.vault.reader import read_note
+from cortex.vault.writer import write_note
+
+
+class KnowledgeCompiler:
+    """Compiles raw sources into structured wiki articles using an LLM."""
+
+    def __init__(self, vault_path: Path) -> None:
+        self.vault_path = vault_path
+        self.client = AsyncOpenAI(
+            api_key=settings.llm_api_key or "unused",
+            base_url=settings.llm_base_url,
+        )
+        self.model = settings.compiler_model
+
+    async def _chat(self, system: str, user_content: str) -> str:
+        """Send a chat completion request and return the assistant's text."""
+        response = await self.client.chat.completions.create(
+            model=self.model,
+            max_tokens=settings.compiler_max_tokens,
+            messages=[
+                {"role": "system", "content": system},
+                {"role": "user", "content": user_content},
+            ],
+        )
+        return response.choices[0].message.content or ""
+
+    async def ingest_source(self, source_path: Path) -> list[Path]:
+        """Read a raw source, call LLM to produce wiki articles, write them to wiki/."""
+        source_content = source_path.read_text()
+        relative_source = str(source_path.relative_to(self.vault_path))
+
+        index_content = self._build_index_context()
+
+        text = await self._chat(
+            INGEST_SYSTEM_PROMPT,
+            f"## Raw Source: {relative_source}\n\n"
+            f"{source_content}\n\n"
+            f"## Existing Wiki Index\n\n{index_content}",
+        )
+
+        articles = self._parse_articles(text)
+
+        created_paths: list[Path] = []
+        for article in articles:
+            filename = article["filename"]
+            if not filename.endswith(".md"):
+                filename += ".md"
+            note_path = self.vault_path / "wiki" / filename
+
+            frontmatter = article.get("frontmatter", {})
+            frontmatter["source_path"] = relative_source
+
+            write_note(
+                path=note_path,
+                vault_root=self.vault_path,
+                content=article["content"],
+                frontmatter=frontmatter,
+                mode="upsert",
+                authored_by=self.model,
+                model=self.model,
+            )
+            created_paths.append(note_path)
+
+        return created_paths
+
+    async def compile_cross_references(self, new_paths: list[Path]) -> None:
+        """After new articles are created, identify cross-references and contradictions."""
+        new_articles = []
+        for p in new_paths:
+            note = read_note(p, self.vault_path)
+            new_articles.append(
+                f"### {note.title}\n"
+                f"Path: {note.path}\n"
+                f"Tags: {', '.join(note.tags)}\n\n"
+                f"{note.content[:500]}"
+            )
+
+        index_context = self._build_index_context()
+
+        text = await self._chat(
+            COMPILE_SYSTEM_PROMPT,
+            f"## New Articles\n\n{'---'.join(new_articles)}\n\n"
+            f"## Existing Wiki Index\n\n{index_context}",
+        )
+
+        updates = self._parse_updates(text)
+        await self._apply_updates(updates)
+
+    def _build_index_context(self) -> str:
+        """Build a summary of existing wiki articles for LLM context."""
+        wiki_dir = self.vault_path / "wiki"
+        if not wiki_dir.exists():
+            return "No existing articles."
+        lines = []
+        for md_file in sorted(wiki_dir.rglob("*.md")):
+            note = read_note(md_file, self.vault_path)
+            tags = ", ".join(note.tags) if note.tags else "none"
+            lines.append(f"- [{note.title}]({note.path}) — tags: {tags}")
+        return "\n".join(lines) if lines else "No existing articles."
+
+    def _parse_articles(self, text: str) -> list[dict]:
+        """Parse LLM response into article dicts, handling code fences."""
+        cleaned = text.strip()
+        if cleaned.startswith("```"):
+            cleaned = cleaned.split("\n", 1)[1]
+        if cleaned.endswith("```"):
+            cleaned = cleaned.rsplit("```", 1)[0]
+        try:
+            return json.loads(cleaned)
+        except json.JSONDecodeError:
+            return []
+
+    def _parse_updates(self, text: str) -> list[dict]:
+        """Parse cross-reference updates from LLM response."""
+        cleaned = text.strip()
+        if cleaned.startswith("```"):
+            cleaned = cleaned.split("\n", 1)[1]
+        if cleaned.endswith("```"):
+            cleaned = cleaned.rsplit("```", 1)[0]
+        try:
+            return json.loads(cleaned)
+        except json.JSONDecodeError:
+            return []
+
+    async def _apply_updates(self, updates: list[dict]) -> None:
+        """Apply cross-reference updates to existing articles."""
+        for update in updates:
+            path = self.vault_path / update["path"]
+            if not path.exists():
+                continue
+            action = update.get("action", "")
+            details = update.get("details", "")
+            if action == "add_link":
+                content = path.read_text()
+                content += f"\n\nSee also: {details}\n"
+                path.write_text(content)
+            elif action == "add_contradiction":
+                content = path.read_text()
+                content += f"\n\n> [!contradiction]\n> {details}\n"
+                path.write_text(content)

cortex/compiler/extractor.py

@@ -0,0 +1,41 @@
+"""Source type detection and text extraction helpers."""
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def detect_source_type(path: Path) -> str:
+    """Detect the type of a raw source file based on its extension."""
+    ext = path.suffix.lower()
+    type_map = {
+        ".pdf": "pdf",
+        ".md": "markdown",
+        ".markdown": "markdown",
+        ".url": "url",
+    }
+    return type_map.get(ext, "text")
+
+
+async def extract_text_from_pdf(path: Path) -> str:
+    """Extract text from a PDF file using pdftotext."""
+    import asyncio
+
+    proc = await asyncio.create_subprocess_exec(
+        "pdftotext", str(path), "-",
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(f"pdftotext error: {stderr.decode()}")
+    return stdout.decode()
+
+
+async def extract_text_from_url(url: str) -> str:
+    """Fetch a URL and return its text content."""
+    import httpx
+
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url, follow_redirects=True)
+        response.raise_for_status()
+        return response.text

cortex/compiler/prompts.py

@@ -0,0 +1,35 @@
+"""System prompts for the knowledge compiler LLM."""
+from __future__ import annotations
+
+INGEST_SYSTEM_PROMPT = """You are a knowledge compiler for a Markdown wiki called Cortex.
+
+Given a raw source document, you must:
+1. Read the source carefully and identify key entities, concepts, claims, and relationships.
+2. Produce one or more structured Markdown wiki articles.
+3. Each article must have YAML frontmatter with: title, tags, authored_by (your model name), created_at, source_path (path to the raw source).
+4. Use [[wikilinks]] to cross-reference other concepts. Link generously.
+5. Flag any claims that might contradict existing knowledge with > [!contradiction] callouts.
+6. Write clearly and concisely. The wiki is for both humans and LLMs to read.
+
+Output format: return a JSON array of objects, each with:
+- "filename": suggested filename (kebab-case, no extension)
+- "frontmatter": YAML frontmatter as a dict
+- "content": Markdown body content
+
+Do NOT include the raw source text verbatim. Synthesize and structure it."""
+
+COMPILE_SYSTEM_PROMPT = """You are maintaining a knowledge wiki called Cortex.
+
+You will receive:
+1. A NEW article that was just created from a raw source.
+2. A list of EXISTING wiki articles (title + path + tags) from the index.
+
+Your job:
+1. Identify which existing articles should be updated with cross-references to the new article.
+2. Identify if any existing claims are contradicted by the new article.
+3. For each article to update, output the specific changes needed.
+
+Output format: return a JSON array of objects, each with:
+- "path": path to the existing article to update
+- "action": "add_link" | "add_contradiction" | "update_content"
+- "details": description of what to add or change"""

cortex/config.py

@@ -0,0 +1,36 @@
+"""Cortex application settings via pydantic-settings."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+
+class CortexSettings(BaseSettings):
+    """Cortex configuration loaded from environment variables with CORTEX_ prefix."""
+    vault_path: Path = Path("./vault")
+
+    qmd_bin: str = "qmd"
+    qmd_url: str = ""
+    qmd_search_mode: str = "keyword"
+
+    llm_api_key: str = ""
+    llm_base_url: str = "https://openrouter.ai/api/v1"
+    compiler_model: str = "anthropic/claude-sonnet-4"
+    compiler_max_tokens: int = 4096
+
+    qmd_refresh_interval_seconds: int = 900
+
+    mcp_transport: str = "stdio"
+    mcp_sse_host: str = "0.0.0.0"
+    mcp_sse_port: int = 8420
+
+    max_context_depth: int = 2
+    max_context_notes: int = 8
+
+    default_author: str = "human"
+
+    model_config = {"env_prefix": "CORTEX_"}
+
+
+settings = CortexSettings()

cortex/graph/builder.py

@@ -0,0 +1,70 @@
+"""Knowledge graph builder — creates nodes and edges from scanned notes."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from cortex.graph.engine import GraphEngine
+from cortex.vault.models import Edge, EdgeType, NodeType, Note
+from cortex.vault.reader import build_wikilink_index, resolve_wikilink
+
+
+async def build_graph(
+    notes: list[Note], graph_path: Path, vault_root: Path
+) -> GraphEngine:
+    """Build or rebuild the knowledge graph from scanned notes.
+
+    1. Add all notes as nodes
+    2. Resolve wikilinks → links_to edges
+    3. Extract tags → tagged_with edges to tag nodes
+    4. Detect source_path → derived_from edges
+    """
+    engine = GraphEngine(graph_path)
+    await engine.load()
+
+    wikilink_index = build_wikilink_index(vault_root)
+
+    async with engine.batch():
+        for note in notes:
+            await engine.add_note_node(note)
+
+        for note in notes:
+            for link in note.wikilinks:
+                resolved = resolve_wikilink(link, vault_root, _index=wikilink_index)
+                if resolved:
+                    await engine.add_edge(
+                        Edge(
+                            source=note.path,
+                            target=resolved,
+                            edge_type=EdgeType.LINKS_TO,
+                        )
+                    )
+
+            for tag in note.tags:
+                tag_id = f"tag:{tag}"
+                if not engine.graph.has_node(tag_id):
+                    engine.graph.add_node(tag_id, node_type=NodeType.TAG.value, title=tag)
+                await engine.add_edge(
+                    Edge(
+                        source=note.path,
+                        target=tag_id,
+                        edge_type=EdgeType.TAGGED_WITH,
+                    )
+                )
+
+            source_path = note.frontmatter.get("source_path")
+            if source_path:
+                if not engine.graph.has_node(source_path):
+                    engine.graph.add_node(
+                        source_path,
+                        node_type=NodeType.RAW_SOURCE.value,
+                        title=source_path,
+                    )
+                await engine.add_edge(
+                    Edge(
+                        source=note.path,
+                        target=source_path,
+                        edge_type=EdgeType.DERIVED_FROM,
+                    )
+                )
+
+    return engine