docgraphical 1.0.0__py3-none-any.whl

docgraph/parser.py

@@ -0,0 +1,209 @@
+"""DocGraph Markdown Parsing & Extraction Engine."""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+@dataclass
+class HeadingNode:
+    level: int
+    title: str
+    line_number: int
+    raw_line: str
+
+
+def parse_headings(file_path: str) -> List[HeadingNode]:
+    """Parse all headings (# ... ######) from a Markdown file, ignoring code blocks."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+        lines = f.readlines()
+
+    headings: List[HeadingNode] = []
+    in_code_block = False
+
+    for idx, line in enumerate(lines, 1):
+        stripped = line.strip()
+
+        # Handle fenced code block guards
+        if stripped.startswith("```") or stripped.startswith("~~~"):
+            in_code_block = not in_code_block
+            continue
+
+        if in_code_block:
+            continue
+
+        # Match ATX headings: # Title
+        m = re.match(r"^(#{1,6})\s+(.+)$", stripped)
+        if m:
+            level = len(m.group(1))
+            title = m.group(2).strip()
+            # Clean trailing hashes if any
+            title = re.sub(r"\s+#+$", "", title)
+            headings.append(HeadingNode(level=level, title=title, line_number=idx, raw_line=line))
+
+    return headings
+
+
+def extract_toc(file_path: str, format_type: str = "text") -> str:
+    """Extract Table of Contents (TOC) with line number anchors.
+
+    format_type: 'text' (indented tree), 'json', or 'markdown'
+    """
+    try:
+        headings = parse_headings(file_path)
+    except Exception as e:
+        return f"Error: {e}"
+
+    if not headings:
+        return f"[DocGraph] No Markdown headings found in {os.path.basename(file_path)}"
+
+    if format_type == "json":
+        import json
+        data = [
+            {"level": h.level, "title": h.title, "line": h.line_number}
+            for h in headings
+        ]
+        return json.dumps(data, indent=2, ensure_ascii=False)
+
+    elif format_type == "markdown":
+        lines = [f"## Table of Contents: {os.path.basename(file_path)}\n"]
+        for h in headings:
+            indent = "  " * (h.level - 1)
+            lines.append(f"{indent}- [{h.title}](#line-{h.line_number}) *(Line {h.line_number})*")
+        return "\n".join(lines)
+
+    else:  # default text tree
+        lines = [f"=== [DocGraph TOC] {os.path.basename(file_path)} ==="]
+        for h in headings:
+            indent = "  " * (h.level - 1)
+            hashes = "#" * h.level
+            lines.append(f"[Line {h.line_number:4d}] {indent}{hashes} {h.title}")
+        return "\n".join(lines)
+
+
+def extract_section(
+    file_path: str,
+    target_heading: str,
+    include_subsections: bool = True
+) -> str:
+    """Surgically extract a specific section by heading title."""
+    if not os.path.exists(file_path):
+        return f"Error: File not found: {file_path}"
+
+    with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+        lines = f.readlines()
+
+    target_clean = target_heading.strip().lower().lstrip("#").strip()
+    matched_idx = -1
+    matched_level = -1
+    in_code_block = False
+
+    # Find the target heading line
+    for idx, line in enumerate(lines):
+        stripped = line.strip()
+        if stripped.startswith("```") or stripped.startswith("~~~"):
+            in_code_block = not in_code_block
+            continue
+        if in_code_block:
+            continue
+
+        m = re.match(r"^(#{1,6})\s+(.+)$", stripped)
+        if m:
+            level = len(m.group(1))
+            title = m.group(2).strip().lower()
+            title = re.sub(r"\s+#+$", "", title)
+            if target_clean in title or target_clean == title:
+                matched_idx = idx
+                matched_level = level
+                break
+
+    if matched_idx == -1:
+        return f"Error: Heading '{target_heading}' not found in {os.path.basename(file_path)}"
+
+    section_lines = [lines[matched_idx]]
+    in_code_block = False
+
+    for idx in range(matched_idx + 1, len(lines)):
+        line = lines[idx]
+        stripped = line.strip()
+
+        if stripped.startswith("```") or stripped.startswith("~~~"):
+            in_code_block = not in_code_block
+            section_lines.append(line)
+            continue
+
+        if not in_code_block:
+            m = re.match(r"^(#{1,6})\s+(.+)$", stripped)
+            if m:
+                level = len(m.group(1))
+                if include_subsections:
+                    if level <= matched_level:
+                        break
+                else:
+                    if level <= matched_level or level > matched_level:
+                        break
+
+        section_lines.append(line)
+
+    result_text = "".join(section_lines).strip()
+    header_banner = f"<!-- [DocGraph Section] {os.path.basename(file_path)} | Line {matched_idx + 1} -->\n"
+    return f"{header_banner}{result_text}"
+
+
+def search_doc(
+    path: str,
+    query: str,
+    max_results: int = 30
+) -> str:
+    """Search Markdown file or directory for relevant sections or lines."""
+    if not os.path.exists(path):
+        return f"Error: Path not found: {path}"
+
+    md_files: List[str] = []
+    if os.path.isfile(path):
+        md_files.append(path)
+    else:
+        for root, _, files in os.walk(path):
+            # Ignore hidden or vendor directories
+            if any(part.startswith(".") or part in ("node_modules", "dist", "build", "__pycache__") for part in root.split(os.sep)):
+                continue
+            for file in files:
+                if file.lower().endswith((".md", ".markdown", ".mdown")):
+                    md_files.append(os.path.join(root, file))
+
+    if not md_files:
+        return f"No Markdown files found in {path}"
+
+    results: List[str] = []
+    query_clean = query.strip().lower()
+
+    for mf in md_files:
+        try:
+            with open(mf, "r", encoding="utf-8", errors="replace") as f:
+                lines = f.readlines()
+
+            for idx, line in enumerate(lines, 1):
+                if query_clean in line.lower():
+                    rel_path = os.path.relpath(mf, path) if os.path.isdir(path) else os.path.basename(mf)
+                    snippet = line.strip()
+                    if len(snippet) > 120:
+                        snippet = snippet[:117] + "..."
+                    results.append(f"- **{rel_path}** [L{idx}]: `{snippet}`")
+                    if len(results) >= max_results:
+                        break
+            if len(results) >= max_results:
+                break
+        except Exception:
+            continue
+
+    if not results:
+        return f"No matches found for '{query}' in {len(md_files)} Markdown file(s)."
+
+    header = f"=== [DocGraph Search] Matches for '{query}' ({len(results)} found) ===\n"
+    return header + "\n".join(results)

docgraph/scanner.py

@@ -0,0 +1,113 @@
+"""Documentation discovery and filesystem tree scanning."""
+import os
+from typing import Dict, List, Any, Optional
+from .constants import IGNORE_DIRS, DOC_EXTS
+from .config import load_config
+from .parser import parse_headings
+
+def is_doc_file(filename: str) -> bool:
+    return filename.lower().endswith(DOC_EXTS)
+
+def scan_doc_repositories(search_roots: List[str]) -> Dict[str, str]:
+    """Discover all folders containing markdown documents under search roots."""
+    cfg = load_config()
+    excluded = [os.path.abspath(p) for p in cfg.get("excluded_paths", [])]
+
+    repos: Dict[str, str] = {}
+
+    for root_dir in search_roots:
+        if not os.path.isdir(root_dir):
+            continue
+        abs_root = os.path.abspath(root_dir)
+        if abs_root in excluded:
+            continue
+
+        # Add root itself
+        repos[os.path.basename(abs_root) or abs_root] = abs_root
+
+        # Scan 2 levels down for project folders
+        try:
+            for entry in os.scandir(abs_root):
+                if entry.is_dir() and entry.name not in IGNORE_DIRS and not entry.name.startswith("."):
+                    sub_path = os.path.abspath(entry.path)
+                    if sub_path not in excluded:
+                        # Check if has any markdown files
+                        has_md = False
+                        try:
+                            for sub_entry in os.scandir(sub_path):
+                                if sub_entry.is_file() and is_doc_file(sub_entry.name):
+                                    has_md = True
+                                    break
+                        except Exception:
+                            pass
+                        if has_md:
+                            repos[entry.name] = sub_path
+        except Exception:
+            pass
+
+    return repos
+
+def get_dir_file_tree(repo_path: str) -> Dict[str, Any]:
+    """Generate recursive folder and markdown file tree."""
+    repo_path = os.path.abspath(repo_path)
+    if not os.path.exists(repo_path):
+        return {"name": os.path.basename(repo_path), "path": repo_path, "type": "dir", "children": []}
+
+    def build_tree(current_path: str) -> Dict[str, Any]:
+        node: Dict[str, Any] = {
+            "name": os.path.basename(current_path) or current_path,
+            "path": current_path,
+            "type": "dir",
+            "children": []
+        }
+        try:
+            entries = sorted(os.scandir(current_path), key=lambda e: (not e.is_dir(), e.name.lower()))
+            for e in entries:
+                if e.name.startswith(".") or e.name in IGNORE_DIRS:
+                    continue
+                if e.is_dir():
+                    child_tree = build_tree(e.path)
+                    # Only include dirs that contain markdown or subdirs
+                    if child_tree["children"]:
+                        node["children"].append(child_tree)
+                elif e.is_file() and is_doc_file(e.name):
+                    try:
+                        headings = parse_headings(e.path)
+                        h_count = len(headings)
+                    except Exception:
+                        h_count = 0
+                    node["children"].append({
+                        "name": e.name,
+                        "path": e.path,
+                        "type": "file",
+                        "headings": h_count,
+                        "size": e.stat().st_size
+                    })
+        except Exception:
+            pass
+        return node
+
+    return build_tree(repo_path)
+
+def get_repo_doc_metrics(repo_path: str) -> Dict[str, Any]:
+    """Compute markdown file count and heading metrics for a repository."""
+    md_files = 0
+    total_headings = 0
+    total_size = 0
+    for root, dirs, files in os.walk(repo_path):
+        dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not d.startswith(".")]
+        for f in files:
+            if is_doc_file(f):
+                md_files += 1
+                full = os.path.join(root, f)
+                try:
+                    total_size += os.path.getsize(full)
+                    h = parse_headings(full)
+                    total_headings += len(h)
+                except Exception:
+                    pass
+    return {
+        "files": md_files,
+        "headings": total_headings,
+        "size": total_size
+    }

docgraph/server.py

@@ -0,0 +1,187 @@
+"""Flask web server application factory and RESTful API routing for DocGraph."""
+import os
+from typing import Optional, List
+from flask import Flask, jsonify, request, Response, render_template
+from .config import load_config, save_config, get_search_roots
+from .scanner import scan_doc_repositories, get_dir_file_tree, get_repo_doc_metrics
+from .parser import extract_toc, extract_section, search_doc, parse_headings
+from .db import index_repository, fetch_graph_data, get_db_path
+
+
+
+
+def create_app(initial_paths: Optional[List[str]] = None, search_roots: Optional[List[str]] = None) -> Flask:
+    """Create and configure the DocGraph Flask application."""
+    if initial_paths and not search_roots:
+        search_roots = initial_paths
+
+    def resolve_file(file_path: str) -> Optional[str]:
+        if not file_path:
+            return None
+        if os.path.isfile(file_path):
+            return os.path.abspath(file_path)
+        roots = get_search_roots(search_roots)
+        for r in roots:
+            cand = os.path.normpath(os.path.join(r, file_path))
+            if os.path.isfile(cand):
+                return cand
+            for dirpath, _, filenames in os.walk(r):
+                if ".git" in dirpath or "node_modules" in dirpath or ".docgraph" in dirpath:
+                    continue
+                for fn in filenames:
+                    full = os.path.join(dirpath, fn)
+                    if fn == file_path or full.replace('\\', '/').endswith(file_path.replace('\\', '/')):
+                        return full
+        return None
+
+    app_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    template_dir = os.path.join(app_root, "templates")
+    static_dir = os.path.join(app_root, "static")
+
+    app = Flask(
+        "docgraph",
+        template_folder=template_dir,
+        static_folder=static_dir
+    )
+
+    @app.after_request
+    def add_security_and_cache_headers(response: Response) -> Response:
+        response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
+        response.headers["Pragma"] = "no-cache"
+        response.headers["Expires"] = "0"
+        return response
+
+    @app.route("/")
+    def index():
+        return render_template("index.html")
+
+    @app.route("/api/projects", methods=["GET"])
+    def get_projects():
+        roots = get_search_roots(search_roots)
+        repos = scan_doc_repositories(roots)
+        result = []
+        for name, p in repos.items():
+            metrics = get_repo_doc_metrics(p)
+            db_file = os.path.join(p, ".docgraph", "docgraph.db")
+            has_db = os.path.exists(db_file)
+            result.append({
+                "name": name,
+                "path": p,
+                "files": metrics["files"],
+                "headings": metrics["headings"],
+                "size": metrics["size"],
+                "has_db": has_db,
+                "status": "ready" if metrics["files"] > 0 else "empty"
+            })
+        return jsonify(result)
+
+    @app.route("/api/graph", methods=["GET"])
+    def get_graph():
+        target_path = request.args.get("path", "").strip()
+        if not target_path or not os.path.exists(target_path):
+            roots = get_search_roots(search_roots)
+            target_path = roots[0] if roots else os.getcwd()
+
+        data = fetch_graph_data(target_path)
+        return jsonify(data)
+
+    @app.route("/api/index", methods=["POST"])
+    def trigger_index():
+        data = request.get_json(silent=True) or {}
+        target_path = data.get("path", "").strip()
+        if not target_path or not os.path.exists(target_path):
+            return jsonify({"success": False, "error": "Invalid project path"}), 400
+
+        f_cnt, n_cnt, e_cnt = index_repository(target_path)
+        return jsonify({"success": True, "files": f_cnt, "nodes": n_cnt, "edges": e_cnt})
+
+    @app.route("/api/tree", methods=["GET"])
+    def get_tree():
+        target_path = request.args.get("path", "").strip()
+        if not target_path or not os.path.exists(target_path):
+            roots = get_search_roots(search_roots)
+            target_path = roots[0] if roots else os.getcwd()
+        tree = get_dir_file_tree(target_path)
+        return jsonify(tree)
+
+    @app.route("/api/doc/toc", methods=["GET"])
+    def get_doc_toc():
+        file_path = request.args.get("file", "").strip()
+        format_type = request.args.get("format", "json").strip()
+        real_path = resolve_file(file_path)
+        if not real_path:
+            return jsonify({"error": "File not found: " + file_path}), 404
+        file_path = real_path
+        toc_output = extract_toc(file_path, format_type=format_type)
+        if format_type == "json":
+            import json
+            try:
+                return jsonify(json.loads(toc_output))
+            except Exception:
+                return jsonify([])
+        return Response(toc_output, mimetype="text/plain")
+
+    @app.route("/api/doc/section", methods=["GET"])
+    def get_doc_section():
+        file_path = request.args.get("file", "").strip()
+        heading = request.args.get("heading", "").strip()
+        include_sub = request.args.get("sub", "1") == "1"
+        real_path = resolve_file(file_path)
+        if not real_path:
+            return jsonify({"error": "File not found: " + file_path}), 404
+        file_path = real_path
+        if not heading:
+            with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+                content = f.read()
+            return jsonify({"heading": "Full Document", "content": content, "sliced": False})
+
+        sec = extract_section(file_path, heading, include_subsections=include_sub)
+        return jsonify({"heading": heading, "content": sec, "sliced": True})
+
+    @app.route("/api/doc/search", methods=["GET"])
+    def search_documents():
+        target_path = request.args.get("path", "").strip()
+        query = request.args.get("q", "").strip()
+        limit = int(request.args.get("limit", 30))
+        if not target_path or not os.path.exists(target_path):
+            roots = get_search_roots(search_roots)
+            target_path = roots[0] if roots else os.getcwd()
+        if not query:
+            return jsonify({"results": []})
+
+        output = search_doc(target_path, query, max_results=limit)
+        return jsonify({"query": query, "output": output})
+
+    @app.route("/api/paths/add", methods=["POST"])
+    def add_custom_path():
+        data = request.get_json(silent=True) or {}
+        new_path = data.get("path", "").strip()
+        if not new_path or not os.path.exists(new_path):
+            return jsonify({"success": False, "error": "Invalid or nonexistent directory path"}), 400
+
+        abs_p = os.path.abspath(new_path)
+        cfg = load_config()
+        if abs_p not in cfg["custom_roots"]:
+            cfg["custom_roots"].append(abs_p)
+            if abs_p in cfg["excluded_paths"]:
+                cfg["excluded_paths"].remove(abs_p)
+            save_config(cfg)
+        return jsonify({"success": True, "path": abs_p})
+
+    @app.route("/api/paths/exclude", methods=["POST"])
+    def exclude_custom_path():
+        data = request.get_json(silent=True) or {}
+        target_path = data.get("path", "").strip()
+        if not target_path:
+            return jsonify({"success": False, "error": "Missing path parameter"}), 400
+
+        abs_p = os.path.abspath(target_path)
+        cfg = load_config()
+        if abs_p in cfg["custom_roots"]:
+            cfg["custom_roots"].remove(abs_p)
+        if abs_p not in cfg["excluded_paths"]:
+            cfg["excluded_paths"].append(abs_p)
+        save_config(cfg)
+        return jsonify({"success": True, "path": abs_p})
+
+    return app

docgraphical/__init__.py

@@ -0,0 +1,13 @@
+# DocGraphical Package Entry
+from .parser import parse_headings, extract_toc, extract_section, search_doc
+
+search_file = search_doc
+
+__version__ = "1.0.0"
+__all__ = [
+    "parse_headings",
+    "extract_toc",
+    "extract_section",
+    "search_doc",
+    "search_file",
+]

docgraphical/cli.py

@@ -0,0 +1,61 @@
+"""DocGraphical Command Line Interface (CLI)."""
+
+import argparse
+import sys
+from docgraph import __version__
+from docgraphical.parser import extract_toc, extract_section, search_doc
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="docgraph",
+        description="DocGraphical: Precision Markdown AST, TOC & Section Slicer for AI Agents & Developers."
+    )
+    parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
+
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    # Command: toc
+    toc_parser = subparsers.add_parser("toc", help="Extract Table of Contents (TOC) with line numbers")
+    toc_parser.add_argument("file", help="Path to the Markdown file")
+    toc_parser.add_argument("--json", action="store_true", help="Output TOC as JSON format")
+    toc_parser.add_argument("--md", action="store_true", help="Output TOC as Markdown list")
+
+    # Command: section
+    sec_parser = subparsers.add_parser("section", help="Extract a specific section by heading")
+    sec_parser.add_argument("file", help="Path to the Markdown file")
+    sec_parser.add_argument("heading", help="Target heading title (e.g. 'Installation' or '## API')")
+    sec_parser.add_argument("--no-sub", action="store_true", help="Exclude subsections under this heading")
+
+    # Command: search
+    search_parser = subparsers.add_parser("search", help="Search keywords across Markdown file or folder")
+    search_parser.add_argument("path", help="Path to Markdown file or root directory")
+    search_parser.add_argument("query", help="Search keyword or term")
+    search_parser.add_argument("--limit", type=int, default=30, help="Maximum number of search results")
+
+    # Command: mcp
+    subparsers.add_parser("mcp", help="Start DocGraphical Model Context Protocol (MCP) server")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.command == "toc":
+        fmt = "json" if args.json else ("markdown" if args.md else "text")
+        print(extract_toc(args.file, format_type=fmt))
+
+    elif args.command == "section":
+        print(extract_section(args.file, args.heading, include_subsections=not args.no_sub))
+
+    elif args.command == "search":
+        print(search_doc(args.path, args.query, max_results=args.limit))
+
+    elif args.command == "mcp":
+        from docgraphical.mcp_server import run_mcp
+        run_mcp()
+
+
+if __name__ == "__main__":
+    main()