diary-docs 0.1.0__py3-none-any.whl

diary/indexer/indexer.py

@@ -0,0 +1,511 @@
+"""Orchestrator — glues scanner, extractors, database, gitignore, and reporter.
+
+Usage::
+
+    from diary.indexer.indexer import run_index, compute_coverage
+    db = run_index(Path("/path/to/workspace"))
+    coverage = compute_coverage(db)
+    db.close()
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+import unicodedata
+from pathlib import Path
+from typing import Optional
+
+from .database import IndexDatabase
+from .extractors import extract_symbols
+from .gitignore import ensure_gitignore
+from .scanner import scan_files, SUPPORTED_EXTENSIONS
+
+logger = logging.getLogger(__name__)
+
+# Extensions that are considered markdown (processed for documents table)
+MARKDOWN_EXTENSIONS = frozenset({".md", ".mdx"})
+
+# Mapping from file extension to human-readable language name
+_EXTENSION_LANGUAGE: dict[str, str] = {
+    ".py": "Python",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".php": "PHP",
+    ".java": "Java",
+    ".go": "Go",
+    ".cs": "C#",
+    ".yaml": "YAML",
+    ".yml": "YAML",
+    ".json": "JSON",
+    ".md": "Markdown",
+    ".mdx": "Markdown",
+}
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────
+
+
+def sha256_hash(content: str | bytes) -> str:
+    """Return the hex SHA-256 digest of *content*.
+
+    Parameters
+    ----------
+    content : str | bytes
+        Input data to hash.
+
+    Returns
+    -------
+    str
+        Hex-encoded SHA-256 digest (64 characters).
+    """
+    if isinstance(content, str):
+        content = content.encode("utf-8")
+    return hashlib.sha256(content).hexdigest()
+
+
+def _extract_frontmatter(content: str) -> dict:
+    """Parse YAML-style frontmatter from markdown content.
+
+    Expects content between the first ``---\\n`` and the second ``\\n---\\n``.
+    Returns a dict with ``title``, ``summary`` and any other key-value pairs
+    found (simple ``key: value`` format only — no nested YAML).
+
+    Parameters
+    ----------
+    content : str
+        Raw markdown file content.
+
+    Returns
+    -------
+    dict
+        Parsed frontmatter fields (may be empty).
+    """
+    result: dict = {}
+    # Match content between first `---\n` and second `\n---\n`
+    m = re.match(r"^---\n(.*?)\n---(?:\n|$)", content, re.DOTALL)
+    if not m:
+        return result
+
+    for line in m.group(1).split("\n"):
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        kv = re.match(r"(\w[\w_-]*)\s*:\s*(.*)", line)
+        if kv:
+            key = kv.group(1).strip()
+            value = kv.group(2).strip()
+            result[key] = value
+
+    return result
+
+
+def _line_count(content: str) -> int:
+    """Return number of lines in *content*."""
+    return len(content.split("\n"))
+
+
+# ── Coverage ─────────────────────────────────────────────────────────────
+
+
+def compute_coverage(db: IndexDatabase) -> dict:
+    """Query *db* and return a dictionary of coverage statistics.
+
+    Parameters
+    ----------
+    db : IndexDatabase
+        An open database handle with populated tables.
+
+    Returns
+    -------
+    dict
+        Keys:
+        - ``total_files``
+        - ``total_symbols``
+        - ``total_documents``
+        - ``documented_files`` (files with at least one relation)
+        - ``documented_symbols`` (symbols referenced in relations)
+        - ``documented_documents`` (documents with at least one relation)
+        - ``files_per_language`` (list of ``{language, count}`` dicts)
+    """
+    conn = db.conn
+
+    def _scalar(sql: str, params: tuple = ()) -> int:
+        row = conn.execute(sql, params).fetchone()
+        return row[0] if row else 0
+
+    total_files = _scalar("SELECT COUNT(*) FROM files")
+    total_symbols = _scalar("SELECT COUNT(*) FROM symbols")
+    total_documents = _scalar("SELECT COUNT(*) FROM documents")
+
+    documented_files = _scalar(
+        "SELECT COUNT(DISTINCT file_id) FROM relations WHERE file_id IS NOT NULL"
+    )
+    documented_symbols = _scalar(
+        "SELECT COUNT(DISTINCT symbol_id) FROM relations"
+    )
+    documented_documents = _scalar(
+        "SELECT COUNT(DISTINCT doc_id) FROM relations"
+    )
+
+    rows_lang = conn.execute(
+        "SELECT language, COUNT(*) FROM files GROUP BY language ORDER BY COUNT(*) DESC"
+    ).fetchall()
+    files_per_language = [{"language": lang, "count": cnt} for lang, cnt in rows_lang]
+
+    return {
+        "total_files": total_files,
+        "total_symbols": total_symbols,
+        "total_documents": total_documents,
+        "documented_files": documented_files,
+        "documented_symbols": documented_symbols,
+        "documented_documents": documented_documents,
+        "files_per_language": files_per_language,
+    }
+
+
+# ── Relation building ────────────────────────────────────────────────────
+
+
+def _build_relations(db: IndexDatabase) -> None:
+    """Link markdown document headings to matching symbol names.
+
+    For every document heading stored in the ``documents`` table, search the
+    ``symbols`` table for a symbol whose name matches (case-insensitive) and
+    insert a relation row with confidence 0.8.
+
+    Parameters
+    ----------
+    db : IndexDatabase
+        Open database handle with populated ``documents`` and ``symbols`` tables.
+    """
+    conn = db.conn
+    docs = conn.execute("SELECT id, headings FROM documents").fetchall()
+
+    for doc_id, headings_json in docs:
+        if not headings_json:
+            continue
+
+        import json as _json
+
+        try:
+            headings = _json.loads(headings_json)
+        except (_json.JSONDecodeError, TypeError):
+            continue
+
+        if not isinstance(headings, list):
+            continue
+
+        for heading in headings:
+            if not isinstance(heading, str) or not heading.strip():
+                continue
+            # Search symbols for case-insensitive match
+            rows = conn.execute(
+                "SELECT id, file_id FROM symbols WHERE LOWER(name) = LOWER(?)",
+                (heading.strip(),),
+            ).fetchall()
+            for sym_id, file_id in rows:
+                db.insert_relation(
+                    doc_id=doc_id,
+                    symbol_id=sym_id,
+                    file_id=file_id,
+                    confidence=0.8,
+                    reason="heading match",
+                )
+
+
+def _build_dependencies(file_path: Path, content: str) -> list[dict]:
+    """Extract import/require statements from *content* using naive regex.
+
+    Supports Python (``import X``, ``from X import Y``), JavaScript/TypeScript
+    (``import X from``, ``require(...)``) and basic static file references.
+
+    Parameters
+    ----------
+    file_path : Path
+        Path to the source file (used for extension detection).
+    content : str
+        Raw file content.
+
+    Returns
+    -------
+    list[dict]
+        Each dict has keys ``target_path`` (the imported module name) and
+        ``dep_type`` (always ``"import"`` for now).
+    """
+    ext = file_path.suffix.lower()
+    deps: list[dict] = []
+
+    if ext == ".py":
+        # Python: import X, from X import Y
+        for m in re.finditer(
+            r"^(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))", content, re.MULTILINE
+        ):
+            target = m.group(1) or m.group(2)
+            deps.append({"target_path": target, "dep_type": "import"})
+    elif ext in (".ts", ".tsx", ".js", ".jsx"):
+        # JS/TS: import X from '...', require('...'), import '...'
+        for m in re.finditer(
+            r'(?:import\s+(?:\w+\s+from\s+)?["\']([^"\']+)["\']|require\(["\']([^"\']+)["\']\))',
+            content,
+        ):
+            target = m.group(1) or m.group(2)
+            deps.append({"target_path": target, "dep_type": "import"})
+    elif ext == ".java":
+        for m in re.finditer(r"^import\s+([\w.*]+);", content, re.MULTILINE):
+            deps.append({"target_path": m.group(1), "dep_type": "import"})
+    elif ext == ".go":
+        for m in re.finditer(r'^import\s+["\']([^"\']+)["\']', content, re.MULTILINE):
+            deps.append({"target_path": m.group(1), "dep_type": "import"})
+    elif ext == ".php":
+        for m in re.finditer(
+            r"(?:use\s+([\w\\\\]+)|require(?:_once)?\s+[\"']([^\"']+)[\"']|include(?:_once)?\s+[\"']([^\"']+)[\"'])",
+            content,
+        ):
+            target = m.group(1) or m.group(2) or m.group(3)
+            deps.append({"target_path": target, "dep_type": "import"})
+
+    return deps
+
+
+# ── Branch name sanitization (ADR-4) ─────────────────────────────────────
+
+
+def _sanitize_branch_name(name: str) -> str:
+    """Sanitize *name* to a safe filesystem fragment.
+
+    * lowercased
+    * ``/`` → ``_``
+    * NFC-normalised
+    * Truncated to 100 code points
+
+    Parameters
+    ----------
+    name : str
+        Raw branch name (e.g. ``feature/my-thing``).
+
+    Returns
+    -------
+    str
+        Safe name usable in a filename (e.g. ``feature_my-thing``).
+    """
+    name = unicodedata.normalize("NFC", name)
+    name = name.lower().replace("/", "_")
+    return name[:100]
+
+
+# ── Main orchestrator ────────────────────────────────────────────────────
+
+
+def run_index(
+    workspace_path: Path,
+    branch_name: str | None = None,
+    db_path: Optional[Path] = None,
+) -> IndexDatabase:
+    """Scan *workspace_path*, extract symbols, build knowledge database.
+
+    This is the main entry point for the knowledge indexer.  It:
+
+    1. Ensures the ``docs/.index/`` output directory exists.
+    2. Updates ``.gitignore`` so the index directory is not tracked.
+    3. Creates / upgrades the SQLite database schema.
+    4. Scans all supported source files.
+    5. Extracts symbols from each file.
+    6. Processes markdown files into a documents table.
+    7. Links documents to symbols via heading matching.
+    8. Extracts import dependency edges.
+
+    Parameters
+    ----------
+    workspace_path : Path
+        Root of the workspace to index.
+    branch_name : str or None, optional
+        If provided, the database is stored as
+        ``docs/.index/knowledge-{sanitized_name}.db``, enabling per-branch
+        isolation.  When ``None`` (default), ``knowledge.db`` is used,
+        preserving backward compatibility.
+    db_path : Path or None, optional
+        Explicit path to the output SQLite database.  If provided, takes
+        precedence over *branch_name*.  If ``None`` (default), the path is
+        derived from *workspace_path* and *branch_name*.
+
+    Returns
+    -------
+    IndexDatabase
+        The populated database instance (still open — caller should
+        ``db.close()`` when done).
+    """
+    workspace_path = workspace_path.resolve()
+
+    # 1. Default database path
+    if db_path is None:
+        index_dir = workspace_path / "docs" / ".index"
+        db_filename = (
+            f"knowledge-{_sanitize_branch_name(branch_name)}.db"
+            if branch_name
+            else "knowledge.db"
+        )
+        db_path = index_dir / db_filename
+    else:
+        index_dir = db_path.parent
+
+    # 2. Ensure output directory exists
+    index_dir.mkdir(parents=True, exist_ok=True)
+
+    # 3. Update .gitignore
+    ensure_gitignore(workspace_path)
+
+    # 4. Initialise database
+    db = IndexDatabase(db_path)
+    db.create_tables()
+    db.clear_all()
+
+    # 5. Scan files
+    all_files = scan_files(workspace_path)
+
+    # Separate source files and markdown files
+    md_files: list[Path] = []
+    source_files: list[Path] = []
+
+    for fp in all_files:
+        if fp.suffix.lower() in MARKDOWN_EXTENSIONS:
+            md_files.append(fp)
+        else:
+            source_files.append(fp)
+
+    # 6. Process source files (populate files + symbols tables)
+    file_count = 0
+    for file_path in source_files:
+        try:
+            content = file_path.read_text("utf-8")
+        except (OSError, UnicodeDecodeError):
+            logger.warning("Cannot read %s — skipping", file_path)
+            continue
+
+        sha256 = sha256_hash(content)
+        rel_path = str(file_path.relative_to(workspace_path).as_posix())
+        ext = file_path.suffix.lower()
+        language = _EXTENSION_LANGUAGE.get(ext, ext.lstrip(".").upper())
+        stat = file_path.stat()
+        lines = _line_count(content)
+
+        file_id = db.insert_file(
+            path=str(file_path),
+            rel_path=rel_path,
+            language=language,
+            sha256=sha256,
+            size=stat.st_size,
+            modified=stat.st_mtime,
+            lines=lines,
+        )
+        file_count += 1
+
+        # Extract symbols
+        symbols = extract_symbols(file_path, content)
+        for sym in symbols:
+            db.insert_symbol(
+                file_id=file_id,
+                name=sym["name"],
+                fqn=sym.get("name"),  # simple FQN = name for v1
+                sym_type=sym["type"],
+                parent=sym.get("parent"),
+                namespace=sym.get("namespace", ""),
+                start_line=sym["line"],
+                end_line=sym.get("end_line", sym["line"]),
+                visibility="public",
+                signature=sym.get("signature", ""),
+            )
+
+        # Extract dependencies
+        deps = _build_dependencies(file_path, content)
+        for dep in deps:
+            db.insert_dependency(
+                source_id=file_id,
+                target_path=dep["target_path"],
+                dep_type=dep["dep_type"],
+            )
+
+    # 7. Process markdown files (populate documents table)
+    for file_path in md_files:
+        try:
+            content = file_path.read_text("utf-8")
+        except (OSError, UnicodeDecodeError):
+            logger.warning("Cannot read %s — skipping", file_path)
+            continue
+
+        sha256 = sha256_hash(content)
+        rel_path = str(file_path.relative_to(workspace_path).as_posix())
+
+        # Extract frontmatter
+        fm = _extract_frontmatter(content)
+        title = fm.get("title", file_path.stem)
+        summary = fm.get("summary", "")
+
+        # Extract headings from markdown symbols
+        md_symbols = extract_symbols(file_path, content)
+        headings = [s["name"] for s in md_symbols if s["type"] == "heading"]
+
+        import json as _json
+
+        headings_json = _json.dumps(headings)
+
+        # Also insert into files table as a regular file
+        stat = file_path.stat()
+        file_id = db.insert_file(
+            path=str(file_path),
+            rel_path=rel_path,
+            language="Markdown",
+            sha256=sha256,
+            size=stat.st_size,
+            modified=stat.st_mtime,
+            lines=_line_count(content),
+        )
+
+        # Insert symbols if any
+        for sym in md_symbols:
+            db.insert_symbol(
+                file_id=file_id,
+                name=sym["name"],
+                fqn=sym["name"],
+                sym_type=sym["type"],
+                parent=sym.get("parent"),
+                namespace=sym.get("namespace", ""),
+                start_line=sym["line"],
+                end_line=sym.get("end_line", sym["line"]),
+            )
+
+        # Insert into documents table
+        db.insert_document(
+            path=rel_path,
+            title=title,
+            headings=headings_json,
+            summary=summary,
+            sha256=sha256,
+        )
+
+    # 8. Build relations
+    _build_relations(db)
+
+    # 9. Store schema version in metadata
+    db.conn.execute(
+        "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)",
+        ("schema_version", "1"),
+    )
+    db.conn.execute(
+        "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)",
+        ("indexed_at", str(db.conn.execute("SELECT strftime('%s','now')").fetchone()[0])),
+    )
+
+    # 10. Final commit
+    db.conn.commit()
+
+    logger.info(
+        "Indexed %d source files and %d markdown files",
+        file_count,
+        len(md_files),
+    )
+
+    return db

diary/indexer/reporter.py

@@ -0,0 +1,137 @@
+"""Rich coverage report for the knowledge index.
+
+Prints a table to stdout summarizing Files, Symbols, Documents,
+and a per-language breakdown.  Read-only — never modifies the database.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+from rich.table import Table
+
+if TYPE_CHECKING:
+    from diary.indexer.database import IndexDatabase
+
+
+def generate_report(db: "IndexDatabase", workspace_path: Path) -> str:
+    """Query *db* for coverage statistics and print a Rich table to stdout.
+
+    Parameters
+    ----------
+    db : IndexDatabase
+        Open database handle (must have tables created).
+    workspace_path : Path
+        Root of the workspace (used only for display, not DB queries).
+
+    Returns
+    -------
+    str
+        The rendered table text (exported from Console).
+    """
+    conn = db.conn
+
+    # ------------------------------------------------------------------
+    # Queries
+    # ------------------------------------------------------------------
+
+    def _scalar(sql: str, params: tuple = ()) -> int:
+        row = conn.execute(sql, params).fetchone()
+        return row[0] if row else 0
+
+    # Summary metrics
+    total_files = _scalar("SELECT COUNT(*) FROM files")
+    documented_files = _scalar(
+        "SELECT COUNT(DISTINCT file_id) FROM relations WHERE file_id IS NOT NULL"
+    )
+
+    total_symbols = _scalar("SELECT COUNT(*) FROM symbols")
+    documented_symbols = _scalar(
+        "SELECT COUNT(DISTINCT symbol_id) FROM relations"
+    )
+
+    total_documents = _scalar("SELECT COUNT(*) FROM documents")
+    documented_documents = _scalar(
+        "SELECT COUNT(DISTINCT doc_id) FROM relations"
+    )
+
+    # Per-language file counts
+    rows_lang = conn.execute(
+        "SELECT language, COUNT(*) FROM files GROUP BY language ORDER BY COUNT(*) DESC"
+    ).fetchall()
+
+    # ------------------------------------------------------------------
+    # Table construction
+    # ------------------------------------------------------------------
+
+    table = Table(
+        title="Knowledge Index Coverage Report",
+        title_justify="left",
+    )
+    table.add_column("Category", style="cyan", no_wrap=True)
+    table.add_column("Total", justify="right")
+    table.add_column("Documented", justify="right")
+    table.add_column("Undocumented", justify="right")
+    table.add_column("Coverage %", justify="right", style="green")
+
+    # -- Summary rows ---------------------------------------------------
+
+    def _pct(documented: int, total: int) -> str:
+        if total == 0:
+            return "—"
+        return f"{documented / total * 100:.1f}%"
+
+    def _undocumented(documented: int, total: int) -> int:
+        return total - documented
+
+    table.add_row(
+        "[bold]Files[/]",
+        str(total_files),
+        str(documented_files),
+        str(_undocumented(documented_files, total_files)),
+        _pct(documented_files, total_files),
+    )
+    table.add_row(
+        "[bold]Symbols[/]",
+        str(total_symbols),
+        str(documented_symbols),
+        str(_undocumented(documented_symbols, total_symbols)),
+        _pct(documented_symbols, total_symbols),
+    )
+    table.add_row(
+        "[bold]Documents[/]",
+        str(total_documents),
+        str(documented_documents),
+        str(_undocumented(documented_documents, total_documents)),
+        _pct(documented_documents, total_documents),
+    )
+
+    # -- Per-Language rows ----------------------------------------------
+    if rows_lang:
+        table.add_section()
+        for language, cnt in rows_lang:
+            expr = """
+                SELECT COUNT(DISTINCT s.id)
+                FROM symbols s
+                JOIN relations r ON r.symbol_id = s.id
+                JOIN files f ON f.id = s.file_id
+                WHERE f.language = ?
+            """
+            docd = _scalar(expr, (language,))
+            table.add_row(
+                f"[italic]{language}[/]",
+                str(cnt),
+                str(docd),
+                str(cnt - docd),
+                _pct(docd, cnt),
+            )
+
+    # ------------------------------------------------------------------
+    # Render & print
+    # ------------------------------------------------------------------
+
+    console = Console(record=True)
+    console.print(table)
+    return console.export_text()

diary/indexer/scanner.py

@@ -0,0 +1,65 @@
+"""File scanner — walks a directory tree and returns supported source files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+SUPPORTED_EXTENSIONS = frozenset({
+    ".php", ".ts", ".js", ".java", ".py",
+    ".go", ".cs", ".yaml", ".yml", ".json", ".md",
+})
+
+EXCLUDED_DIRS = frozenset({
+    "node_modules", ".git", "__pycache__", ".venv", "venv",
+    ".pytest_cache", "dist", "build", ".next", "vendor",
+})
+
+
+def scan_files(root_path: Path, max_file_size: int = 524288) -> list[Path]:
+    """Walk *root_path* and return a sorted list of supported source files.
+
+    Parameters
+    ----------
+    root_path : Path
+        Root directory to scan.
+    max_file_size : int, optional
+        Maximum file size in bytes (default 524288 = 512 KiB).
+
+    Returns
+    -------
+    list[Path]
+        Sorted list of absolute paths matching the supported extensions
+        and exclusion rules.
+    """
+    result: list[Path] = []
+
+    for path in root_path.rglob("*"):
+        # Skip directories – we only collect files
+        if not path.is_file():
+            continue
+
+        # Exclusion: skip if any excluded directory appears in the path
+        if any(part in EXCLUDED_DIRS for part in path.parts):
+            continue
+
+        # Exclusion: skip minified files
+        if ".min." in path.name:
+            continue
+
+        # Inclusion: only accepted extensions
+        if path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+            continue
+
+        # Size check
+        try:
+            if path.stat().st_size > max_file_size:
+                continue
+        except OSError:
+            # Skip files we can't stat (permissions, broken symlinks, etc.)
+            continue
+
+        result.append(path)
+
+    result.sort()
+    return result