docgraphical 1.0.0__py3-none-any.whl

docgraph/__init__.py

@@ -0,0 +1,13 @@
+# DocGraphical Package Entry
+from .parser import parse_headings, extract_toc, extract_section, search_doc
+
+search_file = search_doc
+
+__version__ = "1.0.0"
+__all__ = [
+    "parse_headings",
+    "extract_toc",
+    "extract_section",
+    "search_doc",
+    "search_file",
+]

docgraph/cli.py

@@ -0,0 +1,61 @@
+"""DocGraph Command Line Interface (CLI)."""
+
+import argparse
+import sys
+from docgraph import __version__
+from docgraph.parser import extract_toc, extract_section, search_doc
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="docgraph",
+        description="DocGraph: Precision Markdown AST, TOC & Section Slicer for AI Agents & Developers."
+    )
+    parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
+
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    # Command: toc
+    toc_parser = subparsers.add_parser("toc", help="Extract Table of Contents (TOC) with line numbers")
+    toc_parser.add_argument("file", help="Path to the Markdown file")
+    toc_parser.add_argument("--json", action="store_true", help="Output TOC as JSON format")
+    toc_parser.add_argument("--md", action="store_true", help="Output TOC as Markdown list")
+
+    # Command: section
+    sec_parser = subparsers.add_parser("section", help="Extract a specific section by heading")
+    sec_parser.add_argument("file", help="Path to the Markdown file")
+    sec_parser.add_argument("heading", help="Target heading title (e.g. 'Installation' or '## API')")
+    sec_parser.add_argument("--no-sub", action="store_true", help="Exclude subsections under this heading")
+
+    # Command: search
+    search_parser = subparsers.add_parser("search", help="Search keywords across Markdown file or folder")
+    search_parser.add_argument("path", help="Path to Markdown file or root directory")
+    search_parser.add_argument("query", help="Search keyword or term")
+    search_parser.add_argument("--limit", type=int, default=30, help="Maximum number of search results")
+
+    # Command: mcp
+    subparsers.add_parser("mcp", help="Start DocGraph Model Context Protocol (MCP) server")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.command == "toc":
+        fmt = "json" if args.json else ("markdown" if args.md else "text")
+        print(extract_toc(args.file, format_type=fmt))
+
+    elif args.command == "section":
+        print(extract_section(args.file, args.heading, include_subsections=not args.no_sub))
+
+    elif args.command == "search":
+        print(search_doc(args.path, args.query, max_results=args.limit))
+
+    elif args.command == "mcp":
+        from docgraph.mcp_server import run_mcp
+        run_mcp()
+
+
+if __name__ == "__main__":
+    main()

docgraph/config.py

@@ -0,0 +1,66 @@
+import os
+import sys
+import json
+from typing import Dict, List, Optional
+from .constants import CONFIG_FILE
+
+
+def get_default_repo_root() -> str:
+    """Return default repository root."""
+    return os.getcwd()
+
+
+def get_docgraph_dir(repo_path: Optional[str] = None) -> str:
+    """Return .docgraph directory path."""
+    base = repo_path or get_default_repo_root()
+    return os.path.join(base, ".docgraph")
+
+
+def get_default_db_path(repo_path: Optional[str] = None) -> str:
+    """Return default SQLite database path."""
+    return os.path.join(get_docgraph_dir(repo_path), "docgraph.db")
+
+
+def load_config() -> Dict[str, List[str]]:
+    """Load configuration from user home directory."""
+    if os.path.exists(CONFIG_FILE):
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                roots = data.get("custom_roots") or []
+                excluded = data.get("excluded_paths") or []
+                return {"custom_roots": roots, "excluded_paths": excluded}
+        except Exception:
+            pass
+    return {"custom_roots": [], "excluded_paths": []}
+
+
+def save_config(cfg: Dict[str, List[str]]) -> None:
+    """Save configuration to user home directory."""
+    try:
+        with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+            json.dump(cfg, f, indent=2, ensure_ascii=False)
+    except Exception as e:
+        print(f"Failed to save docgraph config: {e}", file=sys.stderr)
+
+
+def get_search_roots(extra_paths: Optional[List[str]] = None) -> List[str]:
+    """Resolve and deduplicate all documentation search roots."""
+    roots: List[str] = []
+    if extra_paths:
+        for p in extra_paths:
+            abs_p = os.path.abspath(p)
+            if abs_p not in roots and os.path.exists(abs_p):
+                roots.append(abs_p)
+
+    cfg = load_config()
+    for p in cfg.get("custom_roots", []):
+        abs_p = os.path.abspath(p)
+        if abs_p not in roots and os.path.exists(abs_p):
+            roots.append(abs_p)
+
+    # If empty, default to current working directory
+    if not roots:
+        roots.append(os.path.abspath(os.getcwd()))
+
+    return roots

docgraph/constants.py

@@ -0,0 +1,13 @@
+"""Constants and default configuration for DocGraph."""
+import os
+
+IGNORE_DIRS = {
+    ".git", ".codegraph", "node_modules", "dist", "build", ".venv", "venv", "env",
+    "__pycache__", ".pytest_cache", ".mypy_cache", ".idea", ".vscode", "target", "vendor"
+}
+
+DOC_EXTS = (".md", ".markdown", ".mdown", ".txt")
+
+CONFIG_FILE = os.path.expanduser("~/.docgraph_config.json")
+DEFAULT_PORT = 5002
+DEFAULT_HOST = "127.0.0.1"

docgraph/db.py

@@ -0,0 +1,300 @@
+"""DocGraph SQLite Indexing & Graph Storage Engine.
+Stores Markdown files, heading AST nodes, cross-document links, and FTS fulltext search
+inside <project_root>/.docgraph/docgraph.db.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import re
+import sqlite3
+from typing import Any, Dict, List, Optional, Tuple
+from .constants import IGNORE_DIRS, DOC_EXTS
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS files (
+    path TEXT PRIMARY KEY,
+    content_hash TEXT,
+    size INTEGER,
+    modified_at REAL,
+    node_count INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS nodes (
+    id TEXT PRIMARY KEY,
+    file_path TEXT,
+    kind TEXT,            -- 'file', 'heading_1', 'heading_2', 'heading_3', 'code_block'
+    name TEXT,
+    level INTEGER,
+    start_line INTEGER,
+    end_line INTEGER,
+    token_estimate INTEGER,
+    content TEXT,
+    FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS edges (
+    id TEXT PRIMARY KEY,
+    source TEXT,
+    target TEXT,
+    kind TEXT,            -- 'contains', 'parent_child', 'doc_link'
+    line INTEGER,
+    FOREIGN KEY(source) REFERENCES nodes(id) ON DELETE CASCADE
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
+    id UNINDEXED,
+    name,
+    content,
+    tokenize = 'porter unicode61'
+);
+"""
+
+def get_db_path(repo_path: str) -> str:
+    dot_dir = os.path.join(os.path.abspath(repo_path), ".docgraph")
+    os.makedirs(dot_dir, exist_ok=True)
+    return os.path.join(dot_dir, "docgraph.db")
+
+def init_db(db_path: str) -> sqlite3.Connection:
+    conn = sqlite3.connect(db_path)
+    conn.execute("PRAGMA journal_mode=WAL;")
+    conn.execute("PRAGMA foreign_keys=ON;")
+    conn.executescript(SCHEMA_SQL)
+    conn.commit()
+    return conn
+
+def compute_hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest()
+
+def index_repository(repo_path: str) -> Tuple[int, int, int]:
+    repo_path = os.path.abspath(repo_path)
+    db_path = get_db_path(repo_path)
+    conn = init_db(db_path)
+    cur = conn.cursor()
+
+    md_files: List[str] = []
+    for root, dirs, files in os.walk(repo_path):
+        dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not d.startswith(".")]
+        for f in files:
+            if f.lower().endswith(DOC_EXTS):
+                md_files.append(os.path.join(root, f))
+
+    cur.execute("DELETE FROM edges;")
+    cur.execute("DELETE FROM nodes;")
+    cur.execute("DELETE FROM files;")
+    try:
+        cur.execute("DELETE FROM nodes_fts;")
+    except Exception:
+        pass
+
+    total_nodes = 0
+    total_edges = 0
+    pending_edges: List[Tuple[str, str, str, str, int]] = []
+
+    all_rel_files = set(os.path.relpath(f, repo_path).replace("\\", "/") for f in md_files)
+    all_rel_lower = {f.lower(): f for f in all_rel_files}
+
+    # Count unique basenames for cross-doc mention resolution
+    basename_counts: Dict[str, int] = {}
+    for f in all_rel_files:
+        b = os.path.basename(f)
+        basename_counts[b] = basename_counts.get(b, 0) + 1
+    unique_basenames = {b: f for b, f in [(os.path.basename(f), f) for f in all_rel_files] if basename_counts[b] == 1}
+
+    link_pattern = re.compile(r"\[([^\]]+)\]\(([^)#\s]+)(?:#[^)]*)?\)")
+
+    for file_path in md_files:
+        rel_path = os.path.relpath(file_path, repo_path).replace("\\", "/")
+        try:
+            stat = os.stat(file_path)
+            with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+                content = f.read()
+                lines = content.splitlines(keepends=True)
+        except Exception:
+            continue
+
+        c_hash = compute_hash(content)
+        cur.execute(
+            "INSERT INTO files VALUES (?, ?, ?, ?, ?)",
+            (rel_path, c_hash, stat.st_size, stat.st_mtime, 0)
+        )
+
+        file_node_id = f"file::{rel_path}"
+        file_tokens = math.ceil(len(content) / 3.8)
+
+        cur.execute(
+            "INSERT INTO nodes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (file_node_id, rel_path, "file", os.path.basename(rel_path), 0, 1, len(lines), file_tokens, content)
+        )
+        total_nodes += 1
+
+        heading_stack: List[Tuple[int, str]] = [(0, file_node_id)]
+        headings_in_file = 0
+        in_code_block = False
+
+        for idx, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if stripped.startswith("```") or stripped.startswith("~~~"):
+                in_code_block = not in_code_block
+                continue
+
+            if in_code_block:
+                continue
+
+            m = re.match(r"^(#{1,6})\s+(.+)$", stripped)
+            if m:
+                level = len(m.group(1))
+                title = m.group(2).strip()
+                title = re.sub(r"\s+#+$", "", title)
+                node_id = f"heading::{rel_path}::L{idx}::{title[:30]}"
+                headings_in_file += 1
+
+                # Extract section preview snippet
+                sec_lines = [line]
+                for nxt in range(idx, min(len(lines), idx + 20)):
+                    sec_lines.append(lines[nxt])
+                sec_content = "".join(sec_lines)
+
+                cur.execute(
+                    "INSERT INTO nodes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (node_id, rel_path, f"heading_{level}", title, level, idx, idx, math.ceil(len(sec_content)/3.8), sec_content)
+                )
+                total_nodes += 1
+
+                while heading_stack and heading_stack[-1][0] >= level:
+                    heading_stack.pop()
+
+                parent_id = heading_stack[-1][1] if heading_stack else file_node_id
+                edge_id = f"edge::{parent_id}->{node_id}"
+                pending_edges.append((edge_id, parent_id, node_id, "parent_child", idx))
+                total_edges += 1
+
+                heading_stack.append((level, node_id))
+
+        cur.execute("UPDATE files SET node_count = ? WHERE path = ?", (headings_in_file + 1, rel_path))
+
+        # 1. Comprehensive cross-doc Markdown links resolution
+        current_dir = os.path.dirname(rel_path)
+        linked_targets = set()
+
+        for lm in link_pattern.finditer(content):
+            raw_target = lm.group(2).strip().replace("\\", "/")
+            if re.match(r"^(?:https?|mailto|ftp):", raw_target):
+                continue
+
+            candidates = [
+                os.path.normpath(os.path.join(current_dir, raw_target)).replace("\\", "/"),
+                os.path.normpath(os.path.join(current_dir, raw_target + ".md")).replace("\\", "/"),
+                os.path.normpath(os.path.join(current_dir, raw_target, "README.md")).replace("\\", "/"),
+                os.path.normpath(raw_target).replace("\\", "/"),
+                os.path.normpath(raw_target + ".md").replace("\\", "/")
+            ]
+
+            target_match = None
+            for cand in candidates:
+                cand_l = cand.lower()
+                if cand_l in all_rel_lower and all_rel_lower[cand_l] != rel_path:
+                    target_match = all_rel_lower[cand_l]
+                    break
+
+            if target_match and target_match not in linked_targets:
+                linked_targets.add(target_match)
+                link_edge_id = f"link::{file_node_id}->file::{target_match}"
+                pending_edges.append((link_edge_id, file_node_id, f"file::{target_match}", "doc_link", 0))
+                total_edges += 1
+
+        # 2. Textual path and unique doc mentions (forming rich knowledge topology)
+        for other_f in all_rel_files:
+            if other_f != rel_path and other_f not in linked_targets and other_f in content:
+                linked_targets.add(other_f)
+                link_edge_id = f"path::{file_node_id}->file::{other_f}"
+                pending_edges.append((link_edge_id, file_node_id, f"file::{other_f}", "doc_link", 0))
+                total_edges += 1
+
+        for b, target_f in unique_basenames.items():
+            if target_f != rel_path and target_f not in linked_targets and len(b) > 6:
+                if b not in ["README.md", "requirements.txt", "SKILL.md"] and b in content:
+                    linked_targets.add(target_f)
+                    link_edge_id = f"mention::{file_node_id}->file::{target_f}"
+                    pending_edges.append((link_edge_id, file_node_id, f"file::{target_f}", "doc_link", 0))
+                    total_edges += 1
+
+    for e in pending_edges:
+        cur.execute("INSERT OR IGNORE INTO edges VALUES (?, ?, ?, ?, ?)", e)
+
+    try:
+        cur.execute("INSERT INTO nodes_fts(id, name, content) SELECT id, name, content FROM nodes;")
+    except Exception:
+        pass
+
+    conn.commit()
+    conn.close()
+    return len(md_files), total_nodes, total_edges
+
+def fetch_graph_data(repo_path: str) -> Dict[str, Any]:
+    repo_path = os.path.abspath(repo_path)
+    db_path = get_db_path(repo_path)
+    if not os.path.exists(db_path):
+        index_repository(repo_path)
+
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+
+    cur.execute("SELECT id, name, kind, level, start_line, end_line, token_estimate, file_path, content FROM nodes;")
+    rows = cur.fetchall()
+
+    nodes = []
+    KIND_COLORS = {
+        "file": "#f0883e",       # Document (Warm Cyber Orange)
+        "heading_1": "#58a6ff",  # H1 Primary (Electric Blue)
+        "heading_2": "#3fb950",  # H2 Major (Emerald Green)
+        "heading_3": "#bc8cff",  # H3 Subsection (Vivid Purple)
+        "heading_4": "#ff7bba",  # H4 Detail (Vibrant Rose Pink - 100% distinct from Document Orange!)
+        "heading_5": "#00d2d3",  # H5 Fine (Cyan / Turquoise)
+        "heading_6": "#ffd700",  # H6 Micro (Bright Gold)
+    }
+
+    # Hierarchy-based gradual sizing: File(12) -> H1(8.5) -> H2(6.0) -> H3(4.2) -> H4(3.0) -> H5/6(2.2)
+    KIND_VALS = {
+        "file": 12.0,
+        "heading_1": 8.5,
+        "heading_2": 6.0,
+        "heading_3": 4.2,
+        "heading_4": 3.0,
+        "heading_5": 2.2,
+        "heading_6": 1.8
+    }
+
+    for r in rows:
+        nid, name, kind, level, start_l, end_l, tokens, fpath, content = r
+        val = KIND_VALS.get(kind, 3.0)
+        nodes.append({
+            "id": nid,
+            "name": name,
+            "kind": kind,
+            "level": level,
+            "line": start_l,
+            "file": fpath,
+            "abs_path": os.path.normpath(os.path.join(repo_path, fpath)) if not os.path.isabs(fpath) else fpath,
+            "tokens": tokens,
+            "val": val,
+            "color": KIND_COLORS.get(kind, "#8b949e"),
+            "content": content
+        })
+
+    cur.execute("SELECT id, source, target, kind FROM edges;")
+    edges = []
+    for r in cur.fetchall():
+        eid, src, tgt, kind = r
+        edges.append({
+            "id": eid,
+            "source": src,
+            "target": tgt,
+            "kind": kind,
+            "color": "rgba(88, 166, 255, 0.45)" if kind == "parent_child" else "rgba(0, 255, 170, 0.75)"
+        })
+
+    conn.close()
+    return {"nodes": nodes, "links": edges}

docgraph/mcp_server.py

@@ -0,0 +1,69 @@
+"""DocGraph Native Model Context Protocol (MCP) Server.
+Full integration with SQLite Graph Topology (.docgraph/docgraph.db), TOC Slicing, and AST Indexing.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from docgraph.parser import extract_toc, extract_section, search_doc
+from docgraph.db import index_repository, fetch_graph_data, get_db_path
+
+
+def run_mcp():
+    """Start MCP Server using official MCP SDK."""
+    try:
+        from mcp.server import MCPServer
+    except ImportError:
+        print(
+            "Error: 'mcp' SDK is required to run the MCP server.\n"
+            "Install it via: pip install mcp",
+            file=sys.stderr
+        )
+        sys.exit(1)
+
+    app = MCPServer("docgraph")
+
+    @app.tool()
+    def docgraph_toc(filePath: str, format: str = "text") -> str:
+        """Extract Table of Contents (TOC) with line anchors from a Markdown file to save 97% context tokens."""
+        return extract_toc(filePath, format_type=format)
+
+    @app.tool()
+    def docgraph_section(filePath: str, heading: str, includeSubsections: bool = True) -> str:
+        """Surgically extract the content of a specific heading section without reading the whole file."""
+        return extract_section(filePath, heading, include_subsections=includeSubsections)
+
+    @app.tool()
+    def docgraph_search(filePath: str, query: str, limit: int = 30) -> str:
+        """Search across Markdown documents or folders and locate exact line numbers and matches."""
+        return search_doc(filePath, query, max_results=limit)
+
+    @app.tool()
+    def docgraph_graph(repoPath: str) -> str:
+        """Retrieve 3D knowledge topology graph (nodes, edges, cross-document links) from .docgraph/docgraph.db."""
+        try:
+            data = fetch_graph_data(repoPath)
+            return json.dumps({
+                "nodes_count": len(data["nodes"]),
+                "links_count": len(data["links"]),
+                "nodes": data["nodes"][:50],  # summary sample
+                "sample_links": data["links"][:50]
+            }, indent=2, ensure_ascii=False)
+        except Exception as e:
+            return f"Error fetching graph: {e}"
+
+    @app.tool()
+    def docgraph_index(repoPath: str) -> str:
+        """Scan and build/refresh .docgraph/docgraph.db SQLite AST graph index for a project."""
+        try:
+            f, n, e = index_repository(repoPath)
+            return f"Successfully indexed {repoPath}: {f} files, {n} nodes, {e} edges stored in .docgraph/docgraph.db"
+        except Exception as e:
+            return f"Error indexing repository: {e}"
+
+    app.run(transport="stdio")
+
+
+if __name__ == "__main__":
+    run_mcp()