agentramen 0.1.0__py3-none-any.whl

agentramen/__init__.py

@@ -0,0 +1,3 @@
+"""Local-first repository memory and context tools."""
+
+__version__ = "0.1.0"

agentramen/analyzers.py

@@ -0,0 +1,229 @@
+"""Language-specific extraction kept independent from graph storage."""
+
+from __future__ import annotations
+
+import ast
+import importlib
+import posixpath
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Analysis:
+    symbols: tuple[str, ...] = ()
+    imports: tuple[str, ...] = ()
+    calls: tuple[str, ...] = ()
+
+
+class LanguageAnalyzer:
+    language: str
+
+    def analyze(self, path: str, source: str) -> Analysis:
+        raise NotImplementedError
+
+
+class PythonAnalyzer(LanguageAnalyzer):
+    language = "python"
+
+    def analyze(self, path: str, source: str) -> Analysis:
+        try:
+            tree = ast.parse(source, filename=path)
+        except (SyntaxError, ValueError):
+            return Analysis()
+        symbols: set[str] = set()
+        imports: set[str] = set()
+        calls: set[str] = set()
+
+        class Visitor(ast.NodeVisitor):
+            def __init__(self) -> None:
+                self.scope: list[str] = []
+
+            def visit_ClassDef(self, node: ast.ClassDef) -> None:
+                symbols.add(".".join([*self.scope, node.name]))
+                self.scope.append(node.name)
+                self.generic_visit(node)
+                self.scope.pop()
+
+            def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
+                symbols.add(".".join([*self.scope, node.name]))
+                self.scope.append(node.name)
+                self.generic_visit(node)
+                self.scope.pop()
+
+            visit_AsyncFunctionDef = visit_FunctionDef
+
+            def visit_Import(self, node: ast.Import) -> None:
+                imports.update(alias.name for alias in node.names)
+
+            def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+                module = "." * node.level + (node.module or "")
+                imports.add(module)
+                imports.update(f"{module}.{alias.name}" for alias in node.names)
+
+            def visit_Call(self, node: ast.Call) -> None:
+                current = node.func
+                parts: list[str] = []
+                while isinstance(current, ast.Attribute):
+                    parts.append(current.attr)
+                    current = current.value
+                if isinstance(current, ast.Name):
+                    parts.append(current.id)
+                if parts:
+                    calls.add(".".join(reversed(parts)))
+                self.generic_visit(node)
+
+        Visitor().visit(tree)
+        return Analysis(tuple(sorted(symbols)), tuple(sorted(imports)), tuple(sorted(calls)))
+
+
+class RegexAnalyzer(LanguageAnalyzer):
+    """Conservative symbol/import extraction for languages without a built-in parser."""
+
+    def __init__(self, language: str):
+        self.language = language
+        self.symbol_pattern = re.compile(
+            r"^\s*(?:export\s+)?(?:async\s+)?(?:class|interface|type|enum|function|def|fn|func|struct|trait)\s+([A-Za-z_$][\w$]*)",
+            re.MULTILINE,
+        )
+        self.import_pattern = re.compile(
+            r"""^\s*(?:import\s+(?:.*?\s+from\s+)?|from\s+[\w.]+\s+import\s+|(?:const|let|var)\s+\w+\s*=\s*require\()\s*['"]([^'"]+)['"]""",
+            re.MULTILINE,
+        )
+        self.additional_import_patterns = {
+            "java": r"^\s*import\s+(?:static\s+)?([\w.*]+)",
+            "kotlin": r"^\s*import\s+([\w.*]+)",
+            "csharp": r"^\s*using\s+([\w.]+)",
+            "go": r'^\s*import\s+(?:\w+\s+)?["\']([^"\']+)["\']',
+            "rust": r"^\s*use\s+([\w:{},*]+)\s*;",
+            "c": r'^\s*#\s*include\s+[<"]([^>"]+)[>"]',
+            "cpp": r'^\s*#\s*include\s+[<"]([^>"]+)[>"]',
+            "ruby": r"""^\s*require(?:_relative)?\s*\(?\s*['"]([^'"]+)['"]""",
+            "php": r"^\s*use\s+([\w\\]+)",
+            "swift": r"^\s*import\s+([\w.]+)",
+        }
+
+    def analyze(self, path: str, source: str) -> Analysis:
+        symbols = tuple(sorted(set(self.symbol_pattern.findall(source))))
+        imports_found = set(self.import_pattern.findall(source))
+        extra_pattern = self.additional_import_patterns.get(self.language)
+        if extra_pattern:
+            imports_found.update(re.findall(extra_pattern, source, re.MULTILINE))
+        imports = tuple(sorted(imports_found))
+        return Analysis(symbols=symbols, imports=imports)
+
+
+class TreeSitterAnalyzer(LanguageAnalyzer):
+    """Use bundled Tree-sitter grammars when the optional parser extra is installed."""
+
+    _definitions = {
+        "class_declaration", "class_definition", "class_specifier",
+        "function_declaration", "function_definition", "function_item",
+        "method_declaration", "method_definition", "method_item",
+        "interface_declaration", "trait_item", "struct_item", "enum_item",
+        "type_declaration", "type_alias_declaration", "record_declaration",
+    }
+    _calls = {"call_expression", "call", "method_invocation"}
+
+    def __init__(self, language: str):
+        self.language = language
+
+    def analyze(self, path: str, source: str) -> Analysis:
+        try:
+            get_parser = importlib.import_module("tree_sitter_language_pack").get_parser
+            tree = get_parser(self.language).parse(source.encode("utf-8"))
+        except (ImportError, LookupError, RuntimeError, TypeError, ValueError):
+            return RegexAnalyzer(self.language).analyze(path, source)
+
+        symbols: set[str] = set()
+        calls: set[str] = set()
+        source_bytes = source.encode("utf-8")
+        pending = [tree.root_node]
+        while pending:
+            node = pending.pop()
+            pending.extend(reversed(node.children))
+            if node.type in self._definitions:
+                name = node.child_by_field_name("name")
+                if name is None:
+                    name = next(
+                        (
+                            child for child in node.children
+                            if child.type in {
+                                "identifier", "type_identifier", "field_identifier",
+                            }
+                        ),
+                        None,
+                    )
+                if name is not None:
+                    symbols.add(source_bytes[name.start_byte:name.end_byte].decode("utf-8"))
+            if node.type in self._calls:
+                function = node.child_by_field_name("function") or node.child_by_field_name("name")
+                if function is None and node.children:
+                    function = node.children[0]
+                if function is not None:
+                    call = source_bytes[function.start_byte:function.end_byte].decode("utf-8")
+                    if call and len(call) <= 200:
+                        calls.add(call)
+
+        regex_analysis = RegexAnalyzer(self.language).analyze(path, source)
+        return Analysis(
+            symbols=tuple(sorted(symbols)) or regex_analysis.symbols,
+            imports=regex_analysis.imports,
+            calls=tuple(sorted(calls)),
+        )
+
+
+ANALYZERS: dict[str, LanguageAnalyzer] = {
+    "python": PythonAnalyzer(),
+}
+
+
+def analyze(path: str, source: str, language: str | None = None) -> Analysis:
+    language = language or "unknown"
+    analyzer = ANALYZERS.get(
+        language,
+        TreeSitterAnalyzer(language) if language != "unknown" else RegexAnalyzer(language),
+    )
+    return analyzer.analyze(path, source)
+
+
+def module_candidates(
+    import_name: str, source_path: str, paths: set[str] | None = None
+) -> set[str]:
+    """Resolve an import to known source files without assuming one package layout."""
+    if import_name.startswith(("./", "../")):
+        module = posixpath.normpath(
+            posixpath.join(posixpath.dirname(source_path), import_name)
+        )
+        candidates = {
+            f"{module}{suffix}"
+            for suffix in (
+                ".py", ".js", ".jsx", ".ts", ".tsx", ".java", ".go", ".rs",
+                ".c", ".h", ".cpp", ".hpp",
+            )
+        }
+        candidates.update(f"{module}/index{suffix}" for suffix in (".js", ".ts", ".tsx"))
+        return candidates & paths if paths is not None else candidates
+    relative_level = len(import_name) - len(import_name.lstrip("."))
+    module = import_name.lstrip(".").replace(".", "/")
+    if import_name.startswith("."):
+        parent = Path(source_path).parent
+        if relative_level > 1 and len(parent.parents) >= relative_level - 1:
+            parent = parent.parents[relative_level - 2]
+        module = (parent / module).as_posix()
+    candidates = {
+        f"{module}{suffix}"
+        for suffix in (".py", ".js", ".jsx", ".ts", ".tsx", ".java", ".go", ".rs")
+    }
+    candidates.update(f"{module}/__init__.py" for _ in (0,))
+    if paths is None:
+        return candidates
+    candidates.update(
+        path
+        for path in paths
+        if any(path.endswith("/" + module + suffix) for suffix in (
+            ".py", ".js", ".jsx", ".ts", ".tsx", ".java", ".go", ".rs"
+        ))
+    )
+    return candidates & paths

agentramen/benchmark.py

@@ -0,0 +1,227 @@
+"""Reproducible synthetic and temporary-copy repository indexing benchmarks."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from .core import (
+    DEFAULT_EMBEDDING_MODEL,
+    AgentRamenError,
+    LANGUAGES,
+    _terms,
+    context_for,
+    connect,
+    index_repository,
+)
+
+
+def _git(root: Path, *args: str) -> str:
+    return subprocess.run(
+        ["git", "-C", str(root), *args],
+        check=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    ).stdout
+
+
+def _setup_git(root: Path) -> None:
+    subprocess.run(["git", "init", "-q", str(root)], check=True)
+    _git(root, "config", "user.email", "benchmark@example.invalid")
+    _git(root, "config", "user.name", "agentRamen Benchmark")
+
+
+def _measure(
+    root: Path, files: int, task: str, target: Path, semantic: bool = False
+) -> dict[str, object]:
+    if semantic:
+        (root / ".agentramen.yml").write_text(
+            "semantic:\n  enabled: true\n"
+            f"  model: {DEFAULT_EMBEDDING_MODEL}\n",
+            encoding="utf-8",
+        )
+    started = time.perf_counter()
+    initial = index_repository(root)
+    initial_seconds = time.perf_counter() - started
+    target.write_text(
+        target.read_text(encoding="utf-8") + "\n# benchmark incremental edit\n",
+        encoding="utf-8",
+    )
+    started = time.perf_counter()
+    incremental = index_repository(root)
+    incremental_seconds = time.perf_counter() - started
+    started = time.perf_counter()
+    context = context_for(root, task, 500)
+    retrieval_seconds = time.perf_counter() - started
+    terms = _terms(task)
+    conn = connect(root)
+    placeholders = ",".join("?" for _ in terms)
+    candidates = (
+        conn.execute(
+            f"SELECT COUNT(DISTINCT path) FROM file_search_terms "
+            f"WHERE term IN ({placeholders})",
+            sorted(terms),
+        ).fetchone()[0]
+        if terms
+        else 0
+    )
+    database_bytes = sum(
+        path.stat().st_size
+        for path in (root / ".agentramen").glob("graph.db*")
+        if path.is_file()
+    )
+    conn.close()
+    return {
+        "files": files,
+        "task": task,
+        "semantic_enabled": semantic,
+        "initial_index_seconds": round(initial_seconds, 6),
+        "incremental_index_seconds": round(incremental_seconds, 6),
+        "context_retrieval_seconds": round(retrieval_seconds, 6),
+        "initial": initial,
+        "incremental": incremental,
+        "lexical_candidates": candidates,
+        "returned_files": len(context["files"]),
+        "context_estimated_tokens": context["estimated_tokens"],
+        "database_bytes": database_bytes,
+        "note": "Measured locally for this run; not a cross-repository performance claim.",
+    }
+
+
+def run_benchmark(file_count: int, semantic: bool = False) -> dict[str, object]:
+    if file_count < 1:
+        raise ValueError("File count must be positive.")
+    with tempfile.TemporaryDirectory(prefix="agentramen-benchmark-") as directory:
+        root = Path(directory)
+        _setup_git(root)
+        source = root / "src"
+        source.mkdir()
+        for index in range(file_count):
+            identifier = 1000 + index
+            (source / f"service_{identifier}.py").write_text(
+                f"class Service{identifier}:\n"
+                f"    def run(self):\n        return {index}\n",
+                encoding="utf-8",
+            )
+        _git(root, "add", "src")
+        _git(root, "commit", "-qm", "benchmark fixture")
+        identifier = 1000 + file_count - 1
+        target = source / f"service_{identifier}.py"
+        return _measure(root, file_count, str(identifier), target, semantic)
+
+
+def run_repository_benchmark(source_root: Path, semantic: bool = False) -> dict[str, object]:
+    source_root = source_root.resolve()
+    tracked = [path for path in _git(source_root, "ls-files", "-z").split("\0") if path]
+    if not tracked:
+        raise ValueError(f"No tracked files found in {source_root}.")
+    with tempfile.TemporaryDirectory(prefix="agentramen-repository-benchmark-") as directory:
+        root = Path(directory) / "repository"
+        root.mkdir()
+        copied = []
+        for relative in tracked:
+            source = source_root / relative
+            if not source.is_file() or source.is_symlink():
+                continue
+            destination = root / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source, destination)
+            copied.append(relative)
+        eligible = [
+            relative for relative in copied
+            if Path(relative).suffix.lower() in LANGUAGES
+            and (root / relative).stat().st_size <= 2_000_000
+        ]
+        if not eligible:
+            raise ValueError(f"No supported source files found in {source_root}.")
+        target_relative = eligible[0]
+        target = root / target_relative
+        task = Path(target_relative).stem.strip("_").replace("_", " ")
+        _setup_git(root)
+        _git(root, "add", "-A")
+        _git(root, "commit", "-qm", "repository benchmark fixture")
+        return _measure(root, len(eligible), task, target, semantic)
+
+
+def run_comparisons(
+    file_counts: list[int],
+    repository: Path | None,
+    semantic_modes: tuple[bool, ...],
+) -> list[dict[str, object]]:
+    results = []
+    semantic_error: str | None = None
+    for semantic in semantic_modes:
+        scenarios = [
+            (f"synthetic-{count}", lambda count=count: run_benchmark(count, semantic))
+            for count in file_counts
+        ]
+        if repository:
+            scenarios.append(
+                (
+                    f"repository-{repository}",
+                    lambda: run_repository_benchmark(repository, semantic),
+                )
+            )
+        for name, run in scenarios:
+            if semantic and semantic_error:
+                results.append(
+                    {
+                        "scenario": name,
+                        "semantic_enabled": True,
+                        "error": f"Skipped after semantic setup failed: {semantic_error}",
+                    }
+                )
+                continue
+            try:
+                results.append(run())
+            except (AgentRamenError, OSError, ValueError, subprocess.CalledProcessError) as exc:
+                if len(semantic_modes) == 1:
+                    raise
+                if semantic:
+                    semantic_error = str(exc)
+                results.append(
+                    {
+                        "scenario": name,
+                        "semantic_enabled": semantic,
+                        "error": str(exc),
+                    }
+                )
+    return results
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Measure agentRamen indexing and context retrieval")
+    parser.add_argument("--files", nargs="+", type=int, default=[10, 1000])
+    parser.add_argument(
+        "--repository",
+        type=Path,
+        help="Also benchmark a copy of a Git repository's tracked working-tree files",
+    )
+    parser.add_argument(
+        "--semantic-mode",
+        choices=("off", "on", "both"),
+        default="off",
+        help="Benchmark semantic retrieval off, on, or in both modes",
+    )
+    args = parser.parse_args(argv)
+    try:
+        modes = (False, True) if args.semantic_mode == "both" else (
+            args.semantic_mode == "on",
+        )
+        results = run_comparisons(args.files, args.repository, modes)
+    except (OSError, ValueError, subprocess.CalledProcessError) as exc:
+        parser.exit(1, f"agentramen-benchmark: {exc}\n")
+    print(json.dumps(results, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())