diary-docs 0.1.0__py3-none-any.whl

diary/indexer/extractors.py

@@ -0,0 +1,468 @@
+"""
+Regex-based symbol extractors for 10 programming/markup languages.
+
+Limitations (by design):
+ - No syntax validation — purely regex match on lines.
+ - No brace/scope depth tracking — parent assignment uses heuristics
+   (indentation or closest preceding declaration).
+ - Multi-line declarations (e.g. TypeScript generics spanning several lines)
+   are matched on the starting line only.
+ - Nested classes/record-inside-method in brace languages are not tracked
+   — only the first-level nesting is captured for methods inside types.
+
+Language support:
+  .py   — Python
+  .ts   — TypeScript
+  .js   — JavaScript
+  .php  — PHP
+  .java — Java
+  .go   — Go
+  .cs   — C#
+  .yaml / .yml — YAML
+  .json — JSON
+  .md   — Markdown
+"""
+
+import json
+import re
+from pathlib import Path
+
+# ── Type definitions ──────────────────────────────────────────────────
+# Each symbol dict:
+#   name      – identifier name
+#   type      – 'class' | 'function' | 'interface' | 'struct' | 'enum'
+#               | 'trait' | 'record' | 'heading' | 'key'
+#   line      – 1-based line number
+#   parent    – name of enclosing type (or None)
+#   namespace – qualified scope (future use; currently empty)
+#   signature – raw declaration line(s) (currently just the matched line)
+#   end_line  – same as line (no block-size inference)
+
+# ── Helpers ────────────────────────────────────────────────────────────
+
+def _find_parent(lines, type_decls, method_line, method_indent):
+    """
+    Find the most recent type-declaration whose indentation is *strictly*
+    less than *method_indent* and whose line precedes *method_line*.
+    Returns the type name or None.
+    """
+    parent = None
+    for name, _line, _indent in reversed(type_decls):
+        if _line < method_line and _indent < method_indent:
+            parent = name
+            break
+    return parent
+
+
+def _make_symbol(name, typ, line, parent=None, namespace=""):
+    return {
+        "name": name,
+        "type": typ,
+        "line": line,
+        "parent": parent,
+        "namespace": namespace,
+        "signature": "",
+        "end_line": line,
+    }
+
+
+# ── Language handlers ──────────────────────────────────────────────────
+
+def _extract_python(content: str) -> list[dict]:
+    lines = content.split("\n")
+    classes: list[tuple[str, int, int]] = []  # (name, line, indent)
+    symbols: list[dict] = []
+
+    for i, line in enumerate(lines):
+        stripped = line.lstrip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        indent = len(line) - len(stripped)
+
+        cm = re.match(r"class\s+(\w+)", stripped)
+        if cm:
+            classes.append((cm.group(1), i + 1, indent))
+            symbols.append(_make_symbol(cm.group(1), "class", i + 1))
+            continue
+
+        fm = re.match(r"(?:async\s+)?def\s+(\w+)", stripped)
+        if fm:
+            name = fm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, classes, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+
+    return symbols
+
+
+def _extract_typescript(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    type_pattern = re.compile(
+        r"(?:export\s+)?(?:default\s+)?(?:abstract\s+)?"
+        r"(class|interface|type|enum)\s+(\w+)"
+    )
+    func_pattern = re.compile(
+        r"(?:export\s+)?(?:default\s+)?"
+        r"(?:async\s+)?function\s+(\w+)"
+    )
+    # Arrow-function shorthand: `const foo = (...) =>` or `let foo = (...) =>`
+    arrow_pattern = re.compile(r"(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\(")
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("/*"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        tm = type_pattern.search(stripped)
+        if tm:
+            kind, name = tm.group(1), tm.group(2)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, kind, i + 1))
+            continue
+
+        fm = func_pattern.search(stripped)
+        if fm:
+            name = fm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+            continue
+
+        am = arrow_pattern.match(stripped)
+        if am:
+            name = am.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+
+    return symbols
+
+
+def _extract_javascript(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    class_pattern = re.compile(r"class\s+(\w+)")
+    func_pattern = re.compile(
+        r"(?:async\s+)?function\s+(?:\*\s+)?(\w+)"
+    )
+    arrow_pattern = re.compile(
+        r"(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:\(|function)"
+    )
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("/*"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        cm = class_pattern.search(stripped)
+        if cm:
+            name = cm.group(1)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, "class", i + 1))
+            continue
+
+        fm = func_pattern.search(stripped)
+        if fm:
+            name = fm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+            continue
+
+        am = arrow_pattern.match(stripped)
+        if am:
+            name = am.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+
+    return symbols
+
+
+def _extract_php(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    type_pattern = re.compile(
+        r"(?:abstract\s+)?(class|interface|trait)\s+(\w+)"
+    )
+    func_pattern = re.compile(
+        r"(?:public|private|protected)?\s*(?:static\s+)?function\s+(\w+)"
+    )
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("#"):
+            continue
+        if stripped.startswith("<?") or stripped.startswith("//") or stripped.startswith("/*"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        tm = type_pattern.search(stripped)
+        if tm:
+            kind, name = tm.group(1), tm.group(2)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, kind, i + 1))
+            continue
+
+        fm = func_pattern.search(stripped)
+        if fm:
+            name = fm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+
+    return symbols
+
+
+def _extract_java(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    type_pattern = re.compile(
+        r"(?:public|private|protected)?\s*"
+        r"(?:abstract|final|static\s+)?"
+        r"(class|interface|enum)\s+(\w+)"
+    )
+    # Java methods: optional modifiers + return-type + name + (
+    # Use a broad pattern and exclude non-method lines
+    method_pattern = re.compile(
+        r"(?:public|private|protected)?\s*"
+        r"(?:static|final|abstract|synchronized|native|transient|volatile)?\s*"
+        r"(?:<[^>]*>\s*)?"  # generics before return type
+        r"\w+(?:<[^>]*>)?(?:\[\])?\s+"  # return type (simplified)
+        r"(\w+)\s*\("
+    )
+
+    # Keywords that look like method definitions but aren't
+    non_method = {"if", "for", "while", "switch", "catch", "class", "interface", "enum", "return", "new", "this", "super", "throws"}
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("/*") or stripped.startswith("*"):
+            continue
+        if stripped.startswith("import") or stripped.startswith("package"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        tm = type_pattern.search(stripped)
+        if tm:
+            kind, name = tm.group(1), tm.group(2)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, kind, i + 1))
+            continue
+
+        # Check for annotations
+        if stripped.startswith("@"):
+            continue
+
+        mm = method_pattern.search(stripped)
+        if mm and mm.group(1) not in non_method:
+            name = mm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "method", line_no, parent))
+
+    return symbols
+
+
+def _extract_go(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    struct_pattern = re.compile(r"type\s+(\w+)\s+struct")
+    iface_pattern = re.compile(r"type\s+(\w+)\s+interface")
+    # Top-level func: func Name(...)
+    func_pattern = re.compile(r"func\s+(\w+)\s*\(")
+    # Method on type: func (recv *Type|Type) Name(...)
+    method_pattern = re.compile(r"func\s+\([\w\s\*\[\]]+\)\s+(\w+)\s*\(")
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("/*"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        sm = struct_pattern.search(stripped)
+        if sm:
+            name = sm.group(1)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, "struct", i + 1))
+            continue
+
+        im = iface_pattern.search(stripped)
+        if im:
+            name = im.group(1)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, "interface", i + 1))
+            continue
+
+        mm = method_pattern.search(stripped)
+        if mm:
+            name = mm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "function", line_no, parent))
+            continue
+
+        fm = func_pattern.search(stripped)
+        if fm:
+            name = fm.group(1)
+            # Skip if it's actually a method (already matched above)
+            if not stripped.strip().startswith("func ("):
+                line_no = i + 1
+                parent = _find_parent(lines, type_decls, line_no, indent)
+                symbols.append(_make_symbol(name, "function", line_no, parent))
+
+    return symbols
+
+
+def _extract_csharp(content: str) -> list[dict]:
+    lines = content.split("\n")
+    type_decls: list[tuple[str, int, int]] = []
+    symbols: list[dict] = []
+
+    type_pattern = re.compile(
+        r"(?:public|private|protected|internal)?\s*"
+        r"(?:abstract|sealed|static|partial|readonly)?\s*"
+        r"(class|interface|struct|enum|record)\s+(\w+)"
+    )
+    method_pattern = re.compile(
+        r"(?:public|private|protected|internal)?\s*"
+        r"(?:static|virtual|override|abstract|sealed|async|unsafe|new|partial)?\s*"
+        r"\w+(?:<[^>]*>)?(?:\[\])?\s+"  # return type (simplified)
+        r"(\w+)\s*\("
+    )
+
+    non_method = {"if", "for", "while", "switch", "catch", "class", "interface",
+                  "struct", "enum", "record", "return", "new", "this", "base",
+                  "sizeof", "typeof", "nameof", "throw", "yield", "using"}
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("//") or stripped.startswith("/*") or stripped.startswith("*"):
+            continue
+        if stripped.startswith("using ") or stripped.startswith("namespace ") or stripped.startswith("#"):
+            continue
+        indent = len(line) - len(line.lstrip())
+
+        # Skip attributes
+        if stripped.startswith("["):
+            continue
+
+        tm = type_pattern.search(stripped)
+        if tm:
+            kind, name = tm.group(1), tm.group(2)
+            type_decls.append((name, i + 1, indent))
+            symbols.append(_make_symbol(name, kind, i + 1))
+            continue
+
+        mm = method_pattern.search(stripped)
+        if mm and mm.group(1) not in non_method:
+            name = mm.group(1)
+            line_no = i + 1
+            parent = _find_parent(lines, type_decls, line_no, indent)
+            symbols.append(_make_symbol(name, "method", line_no, parent))
+
+    return symbols
+
+
+def _extract_yaml(content: str) -> list[dict]:
+    symbols: list[dict] = []
+    for i, line in enumerate(content.split("\n")):
+        # Top-level key: no leading whitespace, not a comment, not empty
+        if line and not line.startswith(" ") and not line.startswith("\t") and not line.startswith("#"):
+            m = re.match(r"(\w[\w_-]*)\s*:", line)
+            if m:
+                symbols.append(_make_symbol(m.group(1), "key", i + 1))
+    return symbols
+
+
+def _extract_json(content: str) -> list[dict]:
+    try:
+        data = json.loads(content)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(data, dict):
+        return []
+    symbols: list[dict] = []
+    # Find each key in the source to get its line number.
+    for m in re.finditer(r'"([^"]+)"\s*:', content):
+        key = m.group(1)
+        if key in data:
+            line_no = content[: m.start()].count("\n") + 1
+            symbols.append(_make_symbol(key, "key", line_no))
+    return symbols
+
+
+def _extract_markdown(content: str) -> list[dict]:
+    symbols: list[dict] = []
+    pattern = re.compile(r"^(#{1,6})\s+(.+)", re.MULTILINE)
+    for m in pattern.finditer(content):
+        level = len(m.group(1))
+        text = m.group(2).strip()
+        # Calculate 1-based line number
+        line_no = content[: m.start()].count("\n") + 1
+        symbols.append(_make_symbol(text, "heading", line_no))
+    return symbols
+
+
+# ── Extension map ─────────────────────────────────────────────────────
+EXTENSION_MAP: dict[str, callable] = {
+    ".py": _extract_python,
+    ".ts": _extract_typescript,
+    ".tsx": _extract_typescript,
+    ".js": _extract_javascript,
+    ".jsx": _extract_javascript,
+    ".php": _extract_php,
+    ".java": _extract_java,
+    ".go": _extract_go,
+    ".cs": _extract_csharp,
+    ".yaml": _extract_yaml,
+    ".yml": _extract_yaml,
+    ".json": _extract_json,
+    ".md": _extract_markdown,
+    ".mdx": _extract_markdown,
+}
+
+
+# ── Public API ─────────────────────────────────────────────────────────
+
+def extract_symbols(file_path: Path, content: str) -> list[dict]:
+    """
+    Extract symbol declarations from *content* based on the file extension.
+
+    Parameters
+    ----------
+    file_path : Path
+        Path to the source file (used only for its extension).
+    content : str
+        Raw file content (may be empty).
+
+    Returns
+    -------
+    list[dict]
+        Each dict has keys: name, type, line, parent, namespace,
+        signature, end_line.
+        Returns an empty list for unsupported or missing extensions
+        and for empty content.
+    """
+    if not content.strip():
+        return []
+
+    ext = file_path.suffix.lower()
+    handler = EXTENSION_MAP.get(ext)
+    if handler is None:
+        return []
+
+    return handler(content)

diary/indexer/gitignore.py

@@ -0,0 +1,62 @@
+"""gitignore — manage .gitignore entries for the knowledge index directory."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+OLD_ENTRY = "docs/.index/"
+ENTRY = "docs/.index/knowledge-*.db"
+HEADER = "# DIARY Knowledge Index"
+
+
+def ensure_gitignore(root_path: Path) -> bool:
+    """Ensure ``docs/.index/knowledge-*.db`` is listed in ``.gitignore`` at *root_path*.
+
+    If the old entry ``docs/.index/`` exists, it is replaced with the new
+    wildcard pattern ``docs/.index/knowledge-*.db``.
+
+    Parameters
+    ----------
+    root_path : Path
+        Repository root directory that should contain a ``.gitignore``.
+
+    Returns
+    -------
+    bool
+        ``True`` if the file was created or modified, ``False`` if unchanged.
+    """
+    gitignore_path = root_path / ".gitignore"
+
+    if not gitignore_path.exists():
+        gitignore_path.write_text(f"{HEADER}\n{ENTRY}\n")
+        return True
+
+    # Read-modify-write: parse existing content
+    original = gitignore_path.read_text()
+    lines = original.split("\n")
+
+    # Check if the new entry is already present (strip each line for robustness)
+    if any(line.strip() == ENTRY for line in lines):
+        return False
+
+    new_lines: list[str] = []
+    modified = False
+    for line in lines:
+        if line.strip() == OLD_ENTRY:
+            new_lines.append(ENTRY)
+            modified = True
+        else:
+            new_lines.append(line)
+
+    if modified:
+        gitignore_path.write_text("\n".join(new_lines))
+        return True
+
+    # Append — ensure a trailing newline first so we don't graft onto the
+    # last line of an existing file that lacks a final newline.
+    if not original.endswith("\n"):
+        original += "\n"
+
+    gitignore_path.write_text(f"{original}{HEADER}\n{ENTRY}\n")
+    return True