agent-wiki-cli 0.3.28__py3-none-any.whl

llm_wiki_cli/extractors/python_extractor.py

@@ -0,0 +1,346 @@
+"""Python AST extractor for agent-wiki-cli."""
+
+from __future__ import annotations
+
+import ast
+import sys
+from pathlib import Path
+
+from ..config import build_gitignore_matcher
+from .common import discover_source_files
+
+
+# ── AST helper utilities ──────────────────────────────────────────────
+
+
+def _annotation_to_str(node) -> str:
+    """Convert an AST annotation node to a readable string."""
+    if node is None:
+        return ""
+    if isinstance(node, ast.Constant):
+        return repr(node.value)
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        return f"{_annotation_to_str(node.value)}.{node.attr}"
+    if isinstance(node, ast.Subscript):
+        return f"{_annotation_to_str(node.value)}[{_annotation_to_str(node.slice)}]"
+    if isinstance(node, ast.Tuple):
+        return ", ".join(_annotation_to_str(e) for e in node.elts)
+    if isinstance(node, ast.List):
+        return "[" + ", ".join(_annotation_to_str(e) for e in node.elts) + "]"
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
+        return f"{_annotation_to_str(node.left)} | {_annotation_to_str(node.right)}"
+    return ast.dump(node)
+
+
+def _default_to_str(node) -> str:
+    """Convert a default-value AST node to a readable string."""
+    if node is None:
+        return ""
+    if isinstance(node, ast.Constant):
+        return repr(node.value)
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.List):
+        return "[" + ", ".join(_default_to_str(e) for e in node.elts) + "]"
+    if isinstance(node, ast.Dict):
+        return "{...}"
+    if isinstance(node, ast.Call):
+        func = _annotation_to_str(node.func)
+        return f"{func}(...)"
+    return "..."
+
+
+def _extract_decorators(node) -> list[str]:
+    """Extract decorator names from a node."""
+    decorators = []
+    for dec in node.decorator_list:
+        if isinstance(dec, ast.Name):
+            decorators.append(dec.id)
+        elif isinstance(dec, ast.Attribute):
+            decorators.append(_annotation_to_str(dec))
+        elif isinstance(dec, ast.Call):
+            func_str = _annotation_to_str(dec.func)
+            args_parts = []
+            for a in dec.args:
+                args_parts.append(_annotation_to_str(a))
+            for kw in dec.keywords:
+                args_parts.append(f"{kw.arg}={_annotation_to_str(kw.value)}")
+            decorators.append(f"{func_str}({', '.join(args_parts)})")
+    return decorators
+
+
+def _extract_function_info(node) -> dict:
+    """Extract full function/method info from a FunctionDef or AsyncFunctionDef."""
+    info = {
+        "name": node.name,
+        "line": node.lineno,
+        "docstring": ast.get_docstring(node) or "",
+        "decorators": _extract_decorators(node),
+        "is_async": isinstance(node, ast.AsyncFunctionDef),
+    }
+
+    # Parameters (skip 'self'/'cls' for methods)
+    params = []
+    args_node = node.args
+
+    # Pair defaults with args (defaults align to the end of the args list)
+    num_args = len(args_node.args)
+    num_defaults = len(args_node.defaults)
+    default_offset = num_args - num_defaults
+
+    for i, arg in enumerate(args_node.args):
+        if arg.arg in ("self", "cls"):
+            continue
+        param = {
+            "name": arg.arg,
+            "type": _annotation_to_str(arg.annotation),
+        }
+        default_idx = i - default_offset
+        if default_idx >= 0:
+            param["default"] = _default_to_str(args_node.defaults[default_idx])
+        params.append(param)
+
+    info["params"] = params
+    info["return_type"] = _annotation_to_str(node.returns)
+
+    return info
+
+
+def _extract_class_attributes(node) -> list[dict]:
+    """Extract annotated attributes from a class body (Pydantic fields, dataclass fields, etc.)."""
+    attrs = []
+    for child in node.body:
+        if isinstance(child, ast.AnnAssign) and isinstance(child.target, ast.Name):
+            attr = {
+                "name": child.target.id,
+                "type": _annotation_to_str(child.annotation),
+                "default": _default_to_str(child.value) if child.value else "",
+            }
+            attrs.append(attr)
+    return attrs
+
+
+# ── AST visitor ───────────────────────────────────────────────────────
+
+
+class ComponentVisitor(ast.NodeVisitor):
+    def __init__(self, deep: bool = False):
+        self.classes = []
+        self.functions = []  # top-level functions only
+        self.imports = []
+        self.constants = []  # UPPER_CASE module-level assignments
+        self.has_all = False  # whether __all__ is defined
+        self._class_depth = 0
+        self._function_depth = 0
+        self._deep = deep
+
+    def visit_Import(self, node):
+        for alias in node.names:
+            self.imports.append({
+                "module": alias.name,
+                "name": alias.asname or alias.name,
+                "type": "import",
+            })
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node):
+        module = "." * node.level + (node.module or "")
+        for alias in node.names:
+            self.imports.append({
+                "module": module,
+                "name": alias.name,
+                "alias": alias.asname,
+                "type": "from",
+            })
+        self.generic_visit(node)
+
+    def visit_ClassDef(self, node):
+        if self._class_depth > 0 or self._function_depth > 0:
+            return
+
+        bases = [_annotation_to_str(b) for b in node.bases]
+        docstring = ast.get_docstring(node) or ""
+        decorators = _extract_decorators(node)
+        attributes = _extract_class_attributes(node)
+
+        # Extract methods (including private for completeness)
+        methods = []
+        for child in node.body:
+            if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                methods.append(_extract_function_info(child))
+
+        self.classes.append({
+            "name": node.name,
+            "bases": bases,
+            "line": node.lineno,
+            "docstring": docstring,
+            "decorators": decorators,
+            "attributes": attributes,
+            "methods": methods,
+        })
+        # Don't generic_visit — we already walked class body for methods/attrs
+
+    def visit_FunctionDef(self, node):
+        # Only capture top-level functions (not methods inside classes)
+        if self._class_depth == 0 and self._function_depth == 0:
+            if not node.name.startswith("_"):
+                self.functions.append(_extract_function_info(node))
+            elif self._deep:
+                info = _extract_function_info(node)
+                info["private"] = True
+                self.functions.append(info)
+        self._function_depth += 1
+        try:
+            self.generic_visit(node)
+        finally:
+            self._function_depth -= 1
+
+    def visit_AsyncFunctionDef(self, node):
+        if self._class_depth == 0 and self._function_depth == 0:
+            if not node.name.startswith("_"):
+                self.functions.append(_extract_function_info(node))
+            elif self._deep:
+                info = _extract_function_info(node)
+                info["private"] = True
+                self.functions.append(info)
+        self._function_depth += 1
+        try:
+            self.generic_visit(node)
+        finally:
+            self._function_depth -= 1
+
+    def visit_Assign(self, node):
+        """Detect module-level UPPER_CASE constants and ``__all__``."""
+        if self._class_depth == 0 and self._function_depth == 0:
+            for target in node.targets:
+                if isinstance(target, ast.Name):
+                    if target.id == "__all__":
+                        self.has_all = True
+                    elif target.id == target.id.upper() and target.id.replace("_", "").isalnum() and not target.id[0].isdigit():
+                        self.constants.append({
+                            "name": target.id,
+                            "line": node.lineno,
+                        })
+        self.generic_visit(node)
+
+
+# ── Core scan logic ──────────────────────────────────────────────────
+
+
+def _scan_python_files(
+    src_dir: str,
+    deep: bool = False,
+    only_files: list[str] | None = None,
+    include_empty: bool = False,
+) -> dict:
+    """Scan Python files under *src_dir* and return a raw inventory dict.
+
+    The returned dict maps *relative* filepath strings (relative to
+    *src_dir*) to file entry dicts.  The ``"language"`` key is
+    intentionally absent here — callers (e.g. :class:`PythonExtractor`)
+    are responsible for stamping it.
+    """
+    src_path = Path(src_dir).resolve()
+    inventory = {}
+    matcher = build_gitignore_matcher(src_path)
+    py_files = [
+        src_path / rel
+        for rel in discover_source_files(
+            str(src_path), (".py",), only_files=only_files, language="python", matcher=matcher,
+        )
+    ]
+
+    for py_file in py_files:
+        rel = py_file.relative_to(src_path)
+        try:
+            data = py_file.read_bytes()
+            try:
+                source = data.decode("utf-8")
+            except UnicodeDecodeError:
+                source = data.decode("cp1252")
+            tree = ast.parse(source, filename=str(py_file))
+        except UnicodeDecodeError:
+            print(f"llm-wiki Python extractor: skipped undecodable file {rel.as_posix()}", file=sys.stderr)
+            continue
+        except OSError as exc:
+            print(f"llm-wiki Python extractor: failed to read {rel.as_posix()}: {exc}", file=sys.stderr)
+            continue
+        except SyntaxError:
+            continue
+
+        visitor = ComponentVisitor(deep=deep)
+        visitor.visit(tree)
+
+        # Include the file if it has classes, public functions, constants,
+        # __all__, or (in deep mode) private functions.
+        has_content = (
+            visitor.classes
+            or visitor.functions
+            or visitor.constants
+            or visitor.has_all
+            or include_empty
+        )
+        if has_content:
+            file_entry = {
+                "classes": visitor.classes,
+                "functions": visitor.functions,
+            }
+
+            if visitor.constants:
+                file_entry["constants"] = visitor.constants
+            if visitor.has_all:
+                file_entry["has_all"] = True
+
+            if deep:
+                file_entry["imports"] = visitor.imports
+                file_entry["module_docstring"] = ast.get_docstring(tree) or ""
+            else:
+                # Slim format: strip rich fields for backward compat
+                file_entry["classes"] = [
+                    {"name": c["name"], "bases": c["bases"], "line": c["line"]}
+                    for c in visitor.classes
+                ]
+                fns = []
+                for f in visitor.functions:
+                    fn = {"name": f["name"], "line": f["line"]}
+                    if f.get("is_async"):
+                        fn["async"] = True
+                    if f.get("private"):
+                        fn["private"] = True
+                    fns.append(fn)
+                file_entry["functions"] = fns
+
+            inventory[rel.as_posix()] = file_entry
+
+    return inventory
+
+
+# ── Public extractor class ────────────────────────────────────────────
+
+
+class PythonExtractor:
+    """Extractor for Python source files using the built-in :mod:`ast` module.
+
+    Implements :class:`~llm_wiki_cli.extractors.ExtractorProtocol`.
+    """
+
+    def extract(
+        self,
+        src_dir: str,
+        only_files: list[str] | None = None,
+        deep: bool = False,
+        include_empty: bool = False,
+    ) -> dict:
+        """Scan *src_dir* for Python files and return an inventory dict.
+
+        Each file entry includes ``"language": "python"``.
+        """
+        inventory = _scan_python_files(
+            src_dir, deep=deep, only_files=only_files,
+            include_empty=include_empty,
+        )
+        for entry in inventory.values():
+            entry["language"] = "python"
+        return inventory

llm_wiki_cli/extractors/rust_extractor.py

@@ -0,0 +1,143 @@
+"""Rust AST extractor for agent-wiki-cli.
+
+Implements :class:`~llm_wiki_cli.extractors.ExtractorProtocol` by delegating
+to a bundled Rust binary (``rust_scripts/src/main.rs``) that uses the ``syn``
+crate for Rust AST parsing.
+
+Requirements
+------------
+* Rust toolchain (``cargo``) on PATH.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from .common import discover_source_files, filter_bundled_inventory
+
+_RUST_SCRIPTS_DIR = Path(__file__).parent / "rust_scripts"
+
+
+class RustExtractor:
+    """Extractor for Rust source files using a ``cargo run`` subprocess.
+
+    Implements :class:`~llm_wiki_cli.extractors.ExtractorProtocol`.
+
+    Each returned file entry includes ``"language": "rust"``.
+    """
+
+    last_error: str | None = None
+
+    def extract(
+        self,
+        src_dir: str,
+        only_files: list[str] | None = None,
+        deep: bool = False,
+    ) -> dict:
+        """Scan *src_dir* for Rust files and return an inventory dict.
+
+        Parameters
+        ----------
+        src_dir:
+            Root directory to scan.
+        only_files:
+            Optional list of paths (relative to *src_dir*) to restrict
+            extraction to.  When ``None``, all ``.rs`` files found under
+            *src_dir* are scanned (excluding ``target/``, ``vendor/``, etc.).
+        deep:
+            When ``True``, include enriched data (doc comments, struct fields,
+            method details, imports).  When ``False``, return a slim format.
+
+        Returns
+        -------
+        dict
+            ``{filepath: file_entry}`` where each ``file_entry`` contains at
+            minimum ``"classes"``, ``"functions"``, and ``"language"``.
+        """
+        self.last_error = None
+        source_files = discover_source_files(
+            src_dir, (".rs",), only_files=only_files, language="rust",
+        )
+        if not source_files:
+            return {}
+
+        if not shutil.which("cargo"):
+            self.last_error = "cargo not found. Install Rust (https://rustup.rs/) to enable Rust extraction."
+            print(f"llm-wiki Rust extractor: {self.last_error}", file=sys.stderr)
+            return {}
+
+        cmd = [
+            "cargo", "run", "--quiet", "--",
+            "--src-dir", str(Path(src_dir).resolve()),
+        ]
+        cmd += ["--only-files", ",".join(source_files)]
+        if deep:
+            cmd.append("--deep")
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=True,
+                timeout=180,
+                cwd=str(_RUST_SCRIPTS_DIR),
+            )
+        except subprocess.CalledProcessError as exc:
+            self.last_error = "extraction failed"
+            print(
+                f"llm-wiki Rust extractor: extraction failed.\n{exc.stderr}",
+                file=sys.stderr,
+            )
+            return {}
+        except subprocess.TimeoutExpired:
+            self.last_error = "extraction timed out after 180 s"
+            print(
+                "llm-wiki Rust extractor: extraction timed out after 180 s.",
+                file=sys.stderr,
+            )
+            return {}
+        except FileNotFoundError:
+            self.last_error = "cargo executable not found"
+            print(
+                "llm-wiki Rust extractor: cargo executable not found.",
+                file=sys.stderr,
+            )
+            return {}
+
+        # Forward any warnings the Rust script wrote to stderr.
+        if result.stderr.strip():
+            sys.stderr.write(result.stderr)
+
+        if not result.stdout.strip():
+            return {}
+
+        try:
+            inventory: dict = json.loads(result.stdout)
+        except json.JSONDecodeError as exc:
+            self.last_error = "malformed JSON output"
+            print(
+                f"llm-wiki Rust extractor: malformed JSON output — {exc}",
+                file=sys.stderr,
+            )
+            return {}
+
+        for entry in inventory.values():
+            entry["language"] = "rust"
+
+        inventory = filter_bundled_inventory(inventory, _RUST_SCRIPTS_DIR)
+
+        src_root = Path(src_dir).resolve()
+        normalized_inventory: dict = {}
+        for fp, data in inventory.items():
+            try:
+                rel = Path(fp).resolve().relative_to(src_root).as_posix()
+            except ValueError:
+                rel = fp.replace("\\", "/")
+            normalized_inventory[rel] = data
+
+        return normalized_inventory

llm_wiki_cli/extractors/rust_scripts/Cargo.lock

@@ -0,0 +1,110 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "llm-wiki-rust-extractor"
+version = "0.1.0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.149"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"

llm_wiki_cli/extractors/rust_scripts/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "llm-wiki-rust-extractor"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+syn = { version = "2", features = ["full", "extra-traits"] }
+quote = "1"
+proc-macro2 = { version = "1", features = ["span-locations"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"