root-kg 2.0.0__py3-none-any.whl

root_kg/.env.example

@@ -0,0 +1,13 @@
+# ROOT environment variables
+# Copy to .env and fill in your keys
+
+# Option A: Anthropic API (recommended, ~$3 for full extraction)
+# Get key at: https://console.anthropic.com/settings/keys
+ANTHROPIC_API_KEY=sk-ant-your-key-here
+
+# Option B: OpenRouter (free $1 credit)
+# Get key at: https://openrouter.ai/keys
+# OPENROUTER_API_KEY=sk-or-your-key-here
+
+# Option C: Ollama (free, local, no key needed)
+# Just install Ollama and pull a model: ollama pull llama3.1

root_kg/__init__.py

@@ -0,0 +1,3 @@
+"""ROOT: a personal knowledge graph with an MCP server on top."""
+
+__version__ = "2.0.0"

root_kg/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running ROOT as: python -m root_kg <command>"""
+from root_kg.cli import main
+
+if __name__ == "__main__":
+    main()

root_kg/adapters/vault.py

@@ -0,0 +1,122 @@
+"""
+ROOT Obsidian vault adapter.
+
+Reads markdown notes from the vault, extracts metadata,
+and yields them for indexing.
+"""
+
+import hashlib
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator
+
+
+def _extract_title(content: str, filename: str) -> str:
+    """Extract title from frontmatter, first heading, or filename."""
+    # Try YAML frontmatter
+    fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
+    if fm_match:
+        for line in fm_match.group(1).split("\n"):
+            if line.startswith("title:"):
+                return line.split(":", 1)[1].strip().strip("\"'")
+
+    # Try first heading
+    heading_match = re.match(r"^#\s+(.+)", content.lstrip(), re.MULTILINE)
+    if heading_match:
+        return heading_match.group(1).strip()
+
+    # Fall back to filename
+    return filename.replace(".md", "").replace("-", " ").replace("_", " ")
+
+
+def _extract_date(content: str, file_path: Path) -> str | None:
+    """Extract date from frontmatter, filename, or title.
+
+    Supports multiple date formats:
+    - YAML frontmatter: date: or created: field
+    - Filename with dashes: 2026-03-19
+    - Filename/title with spaces: 2026 03 19 (common in Obsidian/Granola)
+    """
+    # Try frontmatter
+    fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
+    if fm_match:
+        for line in fm_match.group(1).split("\n"):
+            if line.startswith("date:") or line.startswith("created:"):
+                date_str = line.split(":", 1)[1].strip().strip("\"'")
+                return date_str
+
+    # Try date in filename (YYYY-MM-DD)
+    date_match = re.search(r"(\d{4}-\d{2}-\d{2})", file_path.name)
+    if date_match:
+        return date_match.group(1)
+
+    # Try date in filename with spaces (YYYY MM DD)
+    space_match = re.search(r"(\d{4})\s+(\d{2})\s+(\d{2})", file_path.stem)
+    if space_match:
+        return f"{space_match.group(1)}-{space_match.group(2)}-{space_match.group(3)}"
+
+    return None
+
+
+def _content_hash(content: str) -> str:
+    """SHA-256 hash of content for change detection."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def _get_folder(path: Path, vault_root: Path) -> str:
+    """Get the top-level folder relative to vault root."""
+    try:
+        relative = path.relative_to(vault_root)
+        parts = relative.parts
+        return parts[0] if len(parts) > 1 else "(root)"
+    except ValueError:
+        return "(unknown)"
+
+
+def scan_vault(
+    vault_path: str | Path,
+    exclude_folders: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
+) -> Iterator[dict]:
+    """Walk the vault and yield note dicts for indexing.
+
+    Yields: {"path": str, "title": str, "content": str,
+             "content_hash": str, "folder": str, "created_at": str|None}
+    """
+    vault_root = Path(vault_path)
+    exclude_folders = set(exclude_folders or [])
+    exclude_patterns = exclude_patterns or []
+
+    if not vault_root.exists():
+        raise FileNotFoundError(f"Vault not found: {vault_root}")
+
+    for md_file in sorted(vault_root.rglob("*.md")):
+        # Skip excluded folders
+        relative_parts = md_file.relative_to(vault_root).parts
+        if any(part in exclude_folders for part in relative_parts):
+            continue
+
+        # Skip excluded patterns
+        if any(md_file.match(pat) for pat in exclude_patterns):
+            continue
+
+        try:
+            content = md_file.read_text(encoding="utf-8", errors="replace")
+        except (OSError, PermissionError):
+            continue
+
+        if not content.strip():
+            continue
+
+        title = _extract_title(content, md_file.name)
+        folder = _get_folder(md_file, vault_root)
+
+        yield {
+            "path": str(md_file.relative_to(vault_root)),
+            "title": title,
+            "content": content,
+            "content_hash": _content_hash(content),
+            "folder": folder,
+            "created_at": _extract_date(content, md_file),
+        }

root_kg/chunker.py

@@ -0,0 +1,53 @@
+"""
+ROOT text chunking.
+
+Strategy: most Obsidian notes are short enough to embed whole.
+Split longer notes at heading boundaries.
+"""
+
+import re
+
+
+MAX_CHUNK_CHARS = 4000  # ~1000 tokens, well within model limits
+
+
+def chunk_size(config: dict) -> int:
+    """Chunk size in characters from config (embeddings.max_chunk_chars), else the default."""
+    return int((config.get("embeddings") or {}).get("max_chunk_chars", MAX_CHUNK_CHARS))
+
+
+def chunk_note(content: str, title: str = "", max_chars: int = MAX_CHUNK_CHARS) -> list[dict]:
+    """Split a note into chunks suitable for embedding.
+
+    Returns list of {"idx": int, "text": str}.
+    Short notes (<max_chars) stay as one chunk.
+    Longer notes split at ## headings.
+    """
+    # Prepend title for context
+    full_text = f"# {title}\n\n{content}" if title else content
+    full_text = full_text.strip()
+
+    if not full_text:
+        return []
+
+    if len(full_text) <= max_chars:
+        return [{"idx": 0, "text": full_text}]
+
+    # Split at headings (## or ###)
+    sections = re.split(r"\n(?=#{1,3}\s)", full_text)
+    chunks = []
+    current_chunk = ""
+    idx = 0
+
+    for section in sections:
+        if len(current_chunk) + len(section) > max_chars and current_chunk:
+            chunks.append({"idx": idx, "text": current_chunk.strip()})
+            idx += 1
+            current_chunk = f"# {title}\n\n" if title else ""
+
+        current_chunk += section + "\n"
+
+    if current_chunk.strip():
+        chunks.append({"idx": idx, "text": current_chunk.strip()})
+
+    return chunks if chunks else [{"idx": 0, "text": full_text[:max_chars]}]

root_kg/cli.py

@@ -0,0 +1,380 @@
+"""
+ROOT CLI.
+
+Setup wizard, management commands, and cron-callable search/ingest
+for the personal knowledge graph.
+
+Usage (cron-safe):
+    root-kg search --query "morning digest signals" [--limit 5]
+    root-kg note --content "Musa pulse: ..." [--tags "musa,signal"]
+
+Exit codes: 0 = success, 1 = error. Errors go to stderr, results to stdout.
+"""
+
+import hashlib
+import os
+import shutil
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+
+from root_kg.paths import PROJECT_ROOT
+CONFIG_PATH = PROJECT_ROOT / "config.yaml"
+ENV_PATH = PROJECT_ROOT / ".env"
+
+# Templates ship inside the package so a PyPI install can run init too.
+PACKAGE_DIR = Path(__file__).resolve().parent
+CONFIG_EXAMPLE = PACKAGE_DIR / "config.example.yaml"
+ENV_EXAMPLE = PACKAGE_DIR / ".env.example"
+
+
+def init():
+    """Interactive setup wizard for ROOT."""
+    print("=" * 50)
+    print("  ROOT: Personal Knowledge Graph Setup")
+    print("=" * 50)
+    print(f"  ROOT home: {PROJECT_ROOT}  (override with ROOT_KG_HOME)")
+    print()
+
+    PROJECT_ROOT.mkdir(parents=True, exist_ok=True)
+
+    # Step 1: Vault path
+    vault_path = _ask_vault_path()
+
+    # Step 2: LLM backend
+    backend, api_key = _ask_llm_backend()
+
+    # Step 3: Create config.yaml
+    _create_config(vault_path, backend)
+
+    # Step 4: Create .env
+    _create_env(backend, api_key)
+
+    # Step 5: Create directories
+    (PROJECT_ROOT / "data").mkdir(exist_ok=True)
+    (PROJECT_ROOT / "logs").mkdir(exist_ok=True)
+
+    print()
+    print("=" * 50)
+    print("  Setup complete!")
+    print("=" * 50)
+    print()
+    print("Next steps:")
+    print("  1. Index your vault:")
+    print("     root-index")
+    print()
+    print("  2. Extract entities (requires LLM):")
+    print("     root-index --extract")
+    print()
+    print("  3. Register as MCP server in Claude Code:")
+    server_bin = shutil.which("root-server") or str(Path(sys.executable).with_name("root-server"))
+    print(f"     claude mcp add root {server_bin}")
+    print()
+    print("  4. Try it:")
+    print('     root_search("your topic")')
+    print('     root_ask("your question")')
+    print()
+
+
+def _ask_vault_path() -> str:
+    """Ask for the vault/notes directory."""
+    print("Where are your notes?")
+    print()
+
+    # Try to detect common vault locations
+    home = Path.home()
+    candidates = [
+        home / "Library/Mobile Documents/iCloud~md~obsidian/Documents",
+        home / "Documents",
+        home / "Obsidian",
+        home / "Notes",
+    ]
+
+    detected = []
+    for candidate in candidates:
+        if candidate.exists():
+            # Look for folders with .md files
+            for folder in candidate.iterdir():
+                if folder.is_dir() and not folder.name.startswith("."):
+                    md_count = len(list(folder.glob("**/*.md")))
+                    if md_count > 10:
+                        detected.append((folder, md_count))
+
+    if detected:
+        print("Detected vaults:")
+        for i, (path, count) in enumerate(detected[:5], 1):
+            print(f"  {i}. {path} ({count} notes)")
+        print(f"  {len(detected[:5]) + 1}. Enter custom path")
+        print()
+
+        choice = input("Choose [1]: ").strip() or "1"
+        try:
+            idx = int(choice) - 1
+            if 0 <= idx < len(detected[:5]):
+                return str(detected[idx][0])
+        except ValueError:
+            pass
+
+    # Custom path
+    while True:
+        path = input("Enter path to your notes folder: ").strip()
+        path = os.path.expanduser(path)
+        if Path(path).exists():
+            md_count = len(list(Path(path).glob("**/*.md")))
+            print(f"  Found {md_count} markdown files.")
+            if md_count > 0:
+                return path
+            print("  No markdown files found. Try a different path.")
+        else:
+            print("  Path doesn't exist. Try again.")
+
+
+def _ask_llm_backend() -> tuple[str, str]:
+    """Ask which LLM backend to use."""
+    print()
+    print("Which LLM backend for entity extraction?")
+    print("  1. Anthropic API (~$3 for full extraction, best quality)")
+    print("  2. OpenRouter (free $1 credit)")
+    print("  3. Ollama (free, local, lower quality)")
+    print("  4. Skip (no extraction, search-only mode)")
+    print()
+
+    choice = input("Choose [1]: ").strip() or "1"
+
+    if choice == "1":
+        key = input("Anthropic API key (or press Enter to add later): ").strip()
+        return "anthropic", key
+    elif choice == "2":
+        key = input("OpenRouter API key (or press Enter to add later): ").strip()
+        return "openrouter", key
+    elif choice == "3":
+        print("  Make sure Ollama is running: ollama serve")
+        print("  And pull a model: ollama pull llama3.1")
+        return "ollama", ""
+    else:
+        return "anthropic", ""
+
+
+def _create_config(vault_path: str, backend: str) -> None:
+    """Create config.yaml from template."""
+    if CONFIG_PATH.exists():
+        overwrite = input(f"\nconfig.yaml already exists. Overwrite? [y/N]: ").strip().lower()
+        if overwrite != "y":
+            print("  Keeping existing config.yaml")
+            return
+
+    with open(CONFIG_EXAMPLE) as f:
+        config = yaml.safe_load(f)
+
+    config["vault"]["path"] = vault_path
+    config["llm"]["backend"] = backend
+
+    # Set model names based on backend
+    if backend == "openrouter":
+        config["llm"]["extraction_model"] = "anthropic/claude-haiku-4-5-20251001"
+        config["llm"]["synthesis_model"] = "anthropic/claude-sonnet-4-20250514"
+    elif backend == "ollama":
+        config["llm"]["extraction_model"] = "llama3.1"
+        config["llm"]["synthesis_model"] = "llama3.1"
+
+    with open(CONFIG_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+
+    print(f"  Created config.yaml")
+
+
+def _create_env(backend: str, api_key: str) -> None:
+    """Create .env file with API key."""
+    if not api_key:
+        if not ENV_PATH.exists():
+            shutil.copy(ENV_EXAMPLE, ENV_PATH)
+            print(f"  Created .env from template (add your API key later)")
+        return
+
+    key_name = {
+        "anthropic": "ANTHROPIC_API_KEY",
+        "openrouter": "OPENROUTER_API_KEY",
+    }.get(backend)
+
+    if key_name:
+        with open(ENV_PATH, "w") as f:
+            f.write(f"{key_name}={api_key}\n")
+        print(f"  Created .env with {key_name}")
+
+
+def stats():
+    """Show ROOT index and graph statistics."""
+    from root_kg.db import RootDB
+    db = RootDB(PROJECT_ROOT / "data/root.db")
+
+    index_stats = db.get_stats()
+    entity_stats = db.get_entity_stats()
+
+    print("ROOT Stats")
+    print(f"  Notes:      {index_stats['total_notes']}")
+    print(f"  Chunks:     {index_stats['total_chunks']}")
+    print(f"  Entities:   {entity_stats['total_entities']}")
+    print(f"  Relations:  {entity_stats['total_relations']}")
+    print(f"  Extracted:  {entity_stats['notes_extracted']}/{index_stats['total_notes']}")
+    print(f"  Last index: {index_stats['last_indexed']}")
+
+    if entity_stats["by_entity_type"]:
+        print("\n  Entity breakdown:")
+        for etype, count in entity_stats["by_entity_type"].items():
+            print(f"    {etype}: {count}")
+
+    db.close()
+
+
+def _load_config() -> dict:
+    """Load config.yaml. Exits with error if missing."""
+    if not CONFIG_PATH.exists():
+        print("Error: config.yaml not found. Run: root-kg init", file=sys.stderr)
+        sys.exit(1)
+    with open(CONFIG_PATH) as f:
+        return yaml.safe_load(f)
+
+
+def search(query: str, limit: int = 5) -> None:
+    """Semantic search across ROOT's knowledge graph.
+
+    Prints one result per line: "<rank>. [<folder>] <title> -- <snippet>"
+    Suitable for capturing into a bash variable.
+    """
+    config = _load_config()
+    db_path = PROJECT_ROOT / config["database"]["path"]
+
+    from root_kg.db import RootDB
+    from root_kg.embeddings import Embedder
+    from root_kg.tools.search import semantic_search
+
+    try:
+        db = RootDB(db_path)
+        embedder = Embedder(config["embeddings"]["model"])
+        results = semantic_search(query=query, db=db, embedder=embedder, limit=limit)
+        db.close()
+    except Exception as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    if not results:
+        # Empty stdout, exit 0 -- caller treats empty output as "no results"
+        return
+
+    for i, r in enumerate(results, 1):
+        snippet = r["snippet"].replace("\n", " ").strip()
+        if len(snippet) > 200:
+            snippet = snippet[:200] + "..."
+        print(f"{i}. [{r['folder']}] {r['title']} -- {snippet}")
+
+
+def note(content: str, tags: list[str] | None = None) -> None:
+    """Ingest a plain-text note into ROOT's index.
+
+    Uses the same chunking + embedding pipeline as root_ingest (server.py).
+    source_type is 'cli' so notes are queryable via root_search without
+    polluting the vault source bucket.
+    """
+    config = _load_config()
+    db_path = PROJECT_ROOT / config["database"]["path"]
+
+    from root_kg.chunker import chunk_note, chunk_size
+    from root_kg.db import RootDB
+    from root_kg.embeddings import Embedder
+
+    now = datetime.now(timezone.utc).isoformat()
+
+    # Derive a short title from the first non-empty line
+    first_line = next((ln.strip() for ln in content.splitlines() if ln.strip()), "CLI Note")
+    title = first_line[:120]
+
+    # Stable unique path: hash of content + ingest timestamp to avoid collisions
+    # on identical content ingested at different times
+    path_hash = hashlib.sha256((content + now).encode()).hexdigest()[:16]
+    path = f"cli-notes/{path_hash}"
+
+    tag_str = (",".join(tags) + " ") if tags else ""
+    # Prepend tags into body so they're searchable
+    indexed_content = f"{tag_str}{content}" if tag_str else content
+    content_hash = hashlib.sha256(indexed_content.encode("utf-8")).hexdigest()
+    folder = "CLI Notes"
+
+    try:
+        db = RootDB(db_path)
+        embedder = Embedder(config["embeddings"]["model"])
+
+        note_id = db.upsert_note(
+            path=path,
+            title=title,
+            content=indexed_content,
+            content_hash=content_hash,
+            folder=folder,
+            source_type="cli",
+            created_at=now,
+            indexed_at=now,
+        )
+
+        chunks = chunk_note(indexed_content, title, max_chars=chunk_size(config))
+        if chunks:
+            texts = [c["text"] for c in chunks]
+            embeddings = embedder.embed_batch(texts)
+            indexed_chunks = [
+                {"idx": c["idx"], "text": c["text"], "embedding": emb}
+                for c, emb in zip(chunks, embeddings)
+            ]
+            db.store_chunks(note_id, indexed_chunks)
+
+        db.close()
+    except Exception as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    # Success: silent stdout, exit 0
+
+
+def main():
+    """CLI entry point."""
+    if len(sys.argv) < 2:
+        print("Usage: root-kg <command>")
+        print("Commands: init, stats, index, extract, search, note")
+        return
+
+    command = sys.argv[1]
+
+    if command == "init":
+        init()
+    elif command == "stats":
+        stats()
+    elif command == "index":
+        from root_kg.indexer import main as index_main
+        sys.argv = sys.argv[1:]  # Shift args for indexer's argparse
+        index_main()
+    elif command == "extract":
+        sys.argv = ["indexer", "--extract-only"] + sys.argv[2:]
+        from root_kg.indexer import main as index_main
+        index_main()
+    elif command == "search":
+        import argparse
+        parser = argparse.ArgumentParser(prog="root-kg search")
+        parser.add_argument("--query", required=True, help="Natural language search query")
+        parser.add_argument("--limit", type=int, default=5, help="Max results (default 5)")
+        args = parser.parse_args(sys.argv[2:])
+        search(query=args.query, limit=args.limit)
+    elif command == "note":
+        import argparse
+        parser = argparse.ArgumentParser(prog="root-kg note")
+        parser.add_argument("--content", required=True, help="Plain text note content")
+        parser.add_argument("--tags", default="", help="Comma-separated tags (optional)")
+        args = parser.parse_args(sys.argv[2:])
+        tags = [t.strip() for t in args.tags.split(",") if t.strip()] if args.tags else []
+        note(content=args.content, tags=tags)
+    else:
+        print(f"Unknown command: {command}")
+        print("Commands: init, stats, index, extract, search, note")
+
+
+if __name__ == "__main__":
+    main()

root_kg/config.example.yaml

@@ -0,0 +1,66 @@
+# ROOT Configuration
+# Copy to config.yaml and edit, or run: root-kg init
+
+vault:
+  # Path to your Obsidian vault (or any folder of markdown files)
+  path: "~/Documents/My Vault"
+  exclude_folders:
+    - ".obsidian"
+    - ".trash"
+    - "templates"
+  exclude_patterns:
+    - "*.excalidraw.md"
+
+  # Extra roots beyond the main vault. Optional; omit for single-vault setups.
+  #
+  # Each root gets its own source_type, so stale sweeps, index stats and the
+  # root_search source filter stay independent per root. An unreachable root is
+  # skipped, never swept.
+  #
+  # `extract` is the cost gate and defaults to false. Indexing is free (local
+  # MiniLM embeddings on CPU), while entity extraction calls an LLM per note.
+  # So a root is semantically searchable the moment it is indexed, and only
+  # earns extraction if you want its entities in the graph. Point a large,
+  # low-entity corpus here with extract: false and it costs you nothing.
+  #
+  # `prefix` namespaces the root's paths, defaulting to its name, because note
+  # paths are unique and two roots can both contain `index.md`. The main vault
+  # above keeps no prefix, so adding roots never re-indexes an existing vault.
+  #
+  # roots:
+  #   - name: project-docs
+  #     path: "~/code/myproject/docs"
+  #     extract: true
+  #   - name: agent-memory
+  #     path: "~/.config/agent/memory"
+  #     extract: false
+
+embeddings:
+  # Local model, free, runs on CPU. No API key needed.
+  model: "all-MiniLM-L6-v2"
+  # Long notes split at headings into chunks of at most this many characters
+  max_chunk_chars: 4000
+
+database:
+  path: "data/root.db"
+
+indexer:
+  log_dir: "logs"
+  batch_size: 64
+  # Skip notes with less prose than this, measured after stripping frontmatter,
+  # headings, embeds and links. Navigation-only notes (a title plus a file embed
+  # plus prev/next links) embed as near-identical tag soup and crowd out real
+  # answers for any query in the same vocabulary. Set 0 to index everything.
+  # Preview before changing: root-index --report-thin --min-prose N
+  min_prose_chars: 30
+
+llm:
+  # Backend options:
+  #   "anthropic" - Direct Anthropic API (needs ANTHROPIC_API_KEY in .env)
+  #   "openrouter" - OpenRouter API (needs OPENROUTER_API_KEY, free $1 credit)
+  #   "ollama" - Local LLM via Ollama (free, no API key needed)
+  backend: "anthropic"
+  extraction_model: "claude-haiku-4-5-20251001"
+  synthesis_model: "claude-sonnet-5"
+  max_content_chars: 6000
+  batch_delay_ms: 100