agentpack-cli 0.1.0__py3-none-any.whl

agentpack/analysis/ranking.py

@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from agentpack.core.models import DependencyGraph, FileInfo
+from agentpack.core.config import ScoringWeights
+
+_STOPWORDS = {
+    "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "is", "are", "was", "were", "be", "been", "have",
+    "has", "had", "do", "does", "did", "will", "would", "could", "should",
+    "may", "might", "can", "that", "this", "these", "those", "it", "its",
+    "from", "not", "we", "our", "you", "your", "they", "them", "their",
+    "file", "files", "code", "function", "method", "class", "module",
+    "use", "using", "used", "how", "what", "when", "where", "why",
+}
+
+_CONCEPT_MAP: dict[str, frozenset[str]] = {
+    # rate limiting
+    "rate": frozenset({"throttle", "ratelimit", "leaky", "bucket", "debounce", "backoff", "quota"}),
+    "limiting": frozenset({"throttle", "ratelimit", "leaky", "bucket", "debounce", "quota"}),
+    "throttle": frozenset({"rate", "limit", "ratelimit", "leaky", "bucket", "quota"}),
+
+    # authentication
+    "auth": frozenset({"jwt", "bearer", "token", "oauth", "credential", "login", "signin", "identity", "principal"}),
+    "authentication": frozenset({"jwt", "bearer", "token", "oauth", "credential", "login"}),
+    "login": frozenset({"auth", "signin", "credential", "token", "session"}),
+
+    # caching
+    "cache": frozenset({"lru", "memoize", "memo", "ttl", "evict", "invalidate", "redis", "memcache"}),
+    "caching": frozenset({"lru", "memoize", "memo", "ttl", "evict", "redis"}),
+
+    # queue / messaging
+    "queue": frozenset({"broker", "pubsub", "kafka", "rabbitmq", "worker", "job", "task", "celery", "enqueue", "dequeue"}),
+    "message": frozenset({"queue", "broker", "pubsub", "event", "dispatch", "emit", "publish", "subscribe"}),
+
+    # database
+    "database": frozenset({"db", "orm", "migration", "schema", "query", "repository", "dao", "entity"}),
+    "db": frozenset({"database", "orm", "migration", "schema", "query", "repository"}),
+    "migration": frozenset({"schema", "database", "db", "alembic", "flyway", "liquibase"}),
+
+    # concurrency
+    "concurrency": frozenset({"mutex", "lock", "semaphore", "atomic", "thread", "async", "goroutine", "coroutine"}),
+    "concurrent": frozenset({"mutex", "lock", "semaphore", "atomic", "thread", "race", "goroutine"}),
+    "race": frozenset({"mutex", "lock", "semaphore", "atomic", "concurrent", "thread"}),
+    "async": frozenset({"await", "coroutine", "future", "promise", "concurrent", "thread"}),
+
+    # error handling
+    "error": frozenset({"exception", "fault", "failure", "retry", "fallback", "circuit", "breaker"}),
+    "retry": frozenset({"backoff", "error", "fault", "resilience", "circuit", "breaker"}),
+
+    # http / api
+    "api": frozenset({"endpoint", "route", "handler", "controller", "rest", "graphql", "grpc", "rpc"}),
+    "endpoint": frozenset({"route", "handler", "controller", "api", "path", "url"}),
+    "middleware": frozenset({"interceptor", "filter", "hook", "plugin", "decorator", "wrapper"}),
+
+    # storage / files
+    "storage": frozenset({"disk", "filesystem", "bucket", "blob", "upload", "download", "s3", "gcs"}),
+    "upload": frozenset({"storage", "blob", "bucket", "multipart", "stream", "file"}),
+
+    # security
+    "security": frozenset({"auth", "permission", "role", "acl", "policy", "rbac", "encrypt", "hash", "sign"}),
+    "permission": frozenset({"role", "acl", "policy", "rbac", "auth", "access", "grant", "deny"}),
+    "encrypt": frozenset({"decrypt", "cipher", "hash", "sign", "verify", "secret", "key"}),
+
+    # logging / observability
+    "log": frozenset({"trace", "metric", "monitor", "observe", "telemetry", "audit", "event"}),
+    "metric": frozenset({"log", "trace", "monitor", "observe", "telemetry", "prometheus", "gauge", "counter"}),
+
+    # search
+    "search": frozenset({"index", "query", "fulltext", "elasticsearch", "solr", "lucene", "rank", "score"}),
+
+    # streaming / SSE / websocket
+    "stream": frozenset({"sse", "websocket", "ws", "chunk", "realtime", "push", "subscribe", "channel"}),
+    "sse": frozenset({"stream", "eventstream", "push", "realtime", "subscribe"}),
+    "websocket": frozenset({"stream", "ws", "socket", "realtime", "channel", "push"}),
+    "realtime": frozenset({"stream", "sse", "websocket", "push", "subscribe", "channel"}),
+
+    # webhooks / events
+    "webhook": frozenset({"event", "callback", "notify", "dispatch", "trigger", "listener", "handler"}),
+    "event": frozenset({"webhook", "listener", "handler", "dispatch", "emit", "publish", "subscribe", "bus"}),
+
+    # pagination
+    "pagination": frozenset({"page", "cursor", "offset", "limit", "paginate", "scroll", "infinite"}),
+    "paginate": frozenset({"page", "cursor", "offset", "limit", "pagination"}),
+
+    # validation / schema
+    "validation": frozenset({"validate", "schema", "sanitize", "constraint", "rule", "pydantic", "zod", "yup"}),
+    "validate": frozenset({"validation", "schema", "sanitize", "constraint"}),
+    "schema": frozenset({"validate", "model", "serializer", "deserializer", "marshal", "unmarshal"}),
+
+    # deployment / infra
+    "deploy": frozenset({"release", "rollout", "container", "docker", "k8s", "kubernetes", "terraform", "ci", "cd"}),
+    "docker": frozenset({"container", "image", "compose", "deploy", "k8s", "registry"}),
+    "kubernetes": frozenset({"k8s", "pod", "deployment", "service", "ingress", "helm", "container"}),
+
+    # email / notifications
+    "email": frozenset({"smtp", "sendgrid", "mailgun", "ses", "template", "notification", "mailer"}),
+    "notification": frozenset({"email", "push", "sms", "alert", "webhook", "event"}),
+
+    # payment / billing
+    "payment": frozenset({"stripe", "paypal", "billing", "invoice", "charge", "subscription", "checkout"}),
+    "billing": frozenset({"payment", "subscription", "invoice", "charge", "plan", "tier"}),
+
+    # file / upload
+    "file": frozenset({"upload", "download", "storage", "s3", "blob", "multipart", "attachment", "disk"}),
+
+    # test / testing
+    "test": frozenset({"spec", "fixture", "mock", "stub", "assert", "expect", "describe", "jest", "pytest"}),
+    "mock": frozenset({"stub", "spy", "patch", "fixture", "test", "fake"}),
+
+    # config / env
+    "config": frozenset({"env", "settings", "environment", "dotenv", "toml", "yaml", "ini", "conf"}),
+    "env": frozenset({"config", "settings", "environment", "dotenv", "variable"}),
+
+    # serialization
+    "serialize": frozenset({"json", "marshal", "encode", "decode", "pickle", "protobuf", "msgpack"}),
+    "deserialize": frozenset({"json", "unmarshal", "decode", "parse", "protobuf"}),
+
+    # health check / liveness
+    "health": frozenset({"ping", "liveness", "readiness", "probe", "heartbeat", "status", "check"}),
+}
+
+_VARIANTS: dict[str, str] = {
+    "cancellation": "cancel",
+    "cancelled": "cancel",
+    "canceling": "cancel",
+    "authentication": "auth",
+    "authenticated": "auth",
+    "authorize": "auth",
+    "authorization": "auth",
+    "configuration": "config",
+    "configured": "config",
+    "database": "db",
+    "databases": "db",
+    "connection": "conn",
+    "connections": "conn",
+    "management": "manage",
+    "manager": "manage",
+    "implementation": "impl",
+    "implements": "impl",
+    "middleware": "middleware",
+    "request": "req",
+    "response": "res",
+    "session": "session",
+    "sessions": "session",
+    "error": "error",
+    "errors": "error",
+    "exception": "exception",
+    "exceptions": "exception",
+    "handler": "handler",
+    "handlers": "handler",
+    "service": "service",
+    "services": "service",
+    "endpoint": "endpoint",
+    "endpoints": "endpoint",
+    "router": "router",
+    "routing": "router",
+    "redis": "redis",
+    "stream": "stream",
+    "streaming": "stream",
+    "goroutine": "goroutine",
+    "channel": "chan",
+    "interface": "interface",
+    "struct": "struct",
+    "trait": "trait",
+    "impl": "impl",
+}
+
+CONFIG_EXTENSIONS = {
+    ".toml", ".yaml", ".yml", ".json", ".env", ".ini", ".cfg", ".conf",
+    ".dockerfile", ".makefile",
+}
+CONFIG_NAMES = {
+    "config", "settings", "configuration", "env", ".env",
+    "pyproject", "package", "dockerfile", "makefile", "cargo", "go",
+    "build", "cmake",
+}
+
+_DEFAULT_WEIGHTS = ScoringWeights()
+
+
+def extract_keywords(task: str) -> set[str]:
+    words = re.split(r"[^a-zA-Z0-9]+", task.lower())
+    keywords: set[str] = set()
+    for word in words:
+        if len(word) < 3:
+            continue
+        if word in _STOPWORDS:
+            continue
+        keywords.add(word)
+        if word in _VARIANTS:
+            keywords.add(_VARIANTS[word])
+
+    # expand via concept map (one level only — no recursion to avoid explosion)
+    expanded: set[str] = set()
+    for kw in keywords:
+        if kw in _CONCEPT_MAP:
+            for synonym in _CONCEPT_MAP[kw]:
+                expanded.add(synonym)
+                # also apply _VARIANTS to expanded terms
+                if synonym in _VARIANTS:
+                    expanded.add(_VARIANTS[synonym])
+    keywords.update(expanded)
+    return keywords
+
+
+def enrich_keywords_from_files(
+    keywords: set[str],
+    changed_paths: set[str],
+    files: list[FileInfo],
+    max_new_keywords: int = 20,
+) -> set[str]:
+    """Expand keywords with high-frequency terms from changed file content.
+
+    Reads only the changed files, extracts identifier-like tokens, and adds
+    those that appear repeatedly — giving the ranker semantic signal beyond
+    the task string alone.
+    """
+    path_map = {fi.path: fi for fi in files if not fi.ignored and not fi.binary}
+    token_freq: dict[str, int] = {}
+
+    for path in changed_paths:
+        fi = path_map.get(path)
+        if fi is None:
+            continue
+        if fi.content is not None:
+            text = fi.content
+        elif fi.abs_path.exists():
+            try:
+                text = fi.abs_path.read_text(errors="replace")
+            except OSError:
+                continue
+        else:
+            continue
+        # Extract camelCase/snake_case identifiers and plain words
+        tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9_]{2,}", text)
+        for raw in tokens:
+            # Split camelCase into parts
+            parts = re.sub(r"([A-Z])", r" \1", raw).lower().split()
+            for part in parts:
+                if len(part) < 3 or part in _STOPWORDS:
+                    continue
+                token_freq[part] = token_freq.get(part, 0) + 1
+
+    # Keep tokens that appear ≥3 times and aren't already in keywords
+    new_keywords = {
+        tok for tok, freq in token_freq.items()
+        if freq >= 3 and tok not in keywords
+    }
+
+    # Limit to top max_new_keywords by frequency
+    top = sorted(new_keywords, key=lambda t: -token_freq[t])[:max_new_keywords]
+    return keywords | set(top)
+
+
+def _path_matches_keywords(path: str, keywords: set[str]) -> bool:
+    path_lower = path.lower()
+    return any(kw in path_lower for kw in keywords)
+
+
+def _content_matches_keywords(text: str, keywords: set[str]) -> int:
+    text_lower = text.lower()
+    return sum(1 for kw in keywords if kw in text_lower)
+
+
+def _symbol_matches_keywords(symbols: list[str], keywords: set[str]) -> bool:
+    for sym in symbols:
+        if any(kw in sym.lower() for kw in keywords):
+            return True
+    return False
+
+
+def score_files(
+    files: list[FileInfo],
+    changed_paths: set[str],
+    staged_paths: set[str],
+    recently_modified: list[str],
+    dep_graph: "DependencyGraph | dict",
+    keywords: set[str],
+    include_tests: bool = True,
+    include_configs: bool = True,
+    weights: ScoringWeights | None = None,
+) -> list[tuple[FileInfo, float, list[str]]]:
+    from agentpack.core.models import DependencyGraph as _DG
+    if not isinstance(dep_graph, _DG):
+        dep_graph = _DG()
+    w = weights or _DEFAULT_WEIGHTS
+    all_paths = {f.path for f in files}
+    results: list[tuple[FileInfo, float, list[str]]] = []
+    recently_set = set(recently_modified[:20])
+
+    for fi in files:
+        if fi.ignored or fi.binary:
+            results.append((fi, w.ignored_penalty, ["ignored/binary"]))
+            continue
+
+        score = 0.0
+        reasons: list[str] = []
+
+        if fi.path in changed_paths:
+            score += w.modified
+            reasons.append("modified")
+
+        if fi.path in staged_paths:
+            score += w.staged
+            reasons.append("staged")
+
+        if _path_matches_keywords(fi.path, keywords):
+            score += w.filename_keyword
+            reasons.append("filename keyword match")
+
+        node = dep_graph.get(fi.path)
+        sym_names: list[str] = []  # symbols aren't stored on DependencyNode; scoring uses path/content only
+        if _symbol_matches_keywords(sym_names, keywords):
+            score += w.symbol_keyword
+            reasons.append("symbol keyword match")
+
+        if fi.content is not None:
+            hits = _content_matches_keywords(fi.content, keywords)
+            if hits > 0:
+                score += min(w.content_keyword_max, hits * w.content_keyword_per_hit)
+                reasons.append(f"content keyword match ({hits})")
+        elif fi.abs_path.exists():
+            try:
+                text = fi.abs_path.read_text(errors="replace")
+                hits = _content_matches_keywords(text, keywords)
+                if hits > 0:
+                    score += min(w.content_keyword_max, hits * w.content_keyword_per_hit)
+                    reasons.append(f"content keyword match ({hits})")
+            except OSError:
+                pass
+
+        for dep_path in node.imports:
+            if dep_path in changed_paths or _path_matches_keywords(dep_path, keywords):
+                score += w.direct_dep
+                reasons.append("direct dependency of changed file")
+                break
+
+        for other_path, other_node in dep_graph.items():
+            if fi.path in other_node.imports and other_path in changed_paths:
+                score += w.reverse_dep
+                reasons.append("reverse dependency")
+                break
+
+        if include_tests:
+            tests = node.tests
+            if tests and any(t in all_paths for t in tests):
+                score += w.related_test
+                reasons.append("has related tests")
+
+            if _is_test_file(fi.path):
+                for src_path in changed_paths:
+                    if _test_matches_source(fi.path, src_path):
+                        score += w.related_test
+                        reasons.append(f"test for {src_path}")
+                        break
+
+        if include_configs and _is_config_file(fi.path):
+            score += w.config_file
+            reasons.append("config file")
+
+        if fi.path in recently_set:
+            score += w.recently_modified
+            reasons.append("recently modified")
+
+        if fi.too_large and score < 50:
+            score += w.large_unrelated_penalty
+            reasons.append("large unrelated file")
+
+        results.append((fi, score, reasons))
+
+    return results
+
+
+def _is_test_file(path: str) -> bool:
+    p = Path(path)
+    return (
+        p.stem.startswith("test_")
+        or p.stem.endswith("_test")
+        or p.stem.endswith(".test")
+        or p.stem.endswith(".spec")
+        or "tests" in p.parts
+        or "__tests__" in p.parts
+    )
+
+
+def _test_matches_source(test_path: str, src_path: str) -> bool:
+    src_stem = Path(src_path).stem
+    test_stem = Path(test_path).stem
+    return src_stem in test_stem or test_stem.replace("test_", "") == src_stem
+
+
+def _is_config_file(path: str) -> bool:
+    p = Path(path)
+    return (
+        p.suffix.lower() in CONFIG_EXTENSIONS
+        or p.stem.lower() in CONFIG_NAMES
+    )

agentpack/analysis/rust_imports.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+# mod foo; / mod foo { — internal module declarations
+_MOD = re.compile(r"^\s*(?:pub\s+)?mod\s+(\w+)\s*[;{]", re.MULTILINE)
+# use foo::bar; / use foo::{bar, baz};
+_USE = re.compile(r"^\s*(?:pub\s+)?use\s+([\w::{}, ]+)\s*;", re.MULTILINE)
+# extern crate foo;
+_EXTERN = re.compile(r"^\s*extern\s+crate\s+(\w+)\s*;", re.MULTILINE)
+
+
+def extract_imports(path: Path, text: str | None = None) -> list[str]:
+    if text is None:
+        try:
+            text = path.read_text(errors="replace")
+        except OSError:
+            return []
+
+    imports: list[str] = []
+    for m in _MOD.finditer(text):
+        imports.append(m.group(1))
+    for m in _USE.finditer(text):
+        # strip whitespace and braces for a clean root crate name
+        raw = m.group(1).split("::")[0].strip()
+        if raw:
+            imports.append(raw)
+    for m in _EXTERN.finditer(text):
+        imports.append(m.group(1))
+
+    return list(dict.fromkeys(imports))  # deduplicate, preserve order

agentpack/analysis/symbols.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+from typing import Literal
+
+from agentpack.core.models import Symbol
+
+
+def extract_python_symbols(path: Path) -> list[Symbol]:
+    try:
+        source = path.read_text(errors="replace")
+        tree = ast.parse(source)
+    except (SyntaxError, OSError):
+        return []
+
+    symbols: list[Symbol] = []
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            sig = f"class {node.name}"
+            if node.bases:
+                bases = ", ".join(ast.unparse(b) for b in node.bases)
+                sig += f"({bases})"
+            doc = ast.get_docstring(node)
+            symbols.append(
+                Symbol(
+                    name=node.name,
+                    kind="class",
+                    start_line=node.lineno,
+                    end_line=node.end_lineno or node.lineno,
+                    signature=sig,
+                    summary=doc[:120] if doc else None,
+                    body=ast.get_source_segment(source, node),
+                )
+            )
+            for item in node.body:
+                if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    msig = f"def {item.name}({_args_str(item.args)})"
+                    mdoc = ast.get_docstring(item)
+                    symbols.append(
+                        Symbol(
+                            name=f"{node.name}.{item.name}",
+                            kind="method",
+                            start_line=item.lineno,
+                            end_line=item.end_lineno or item.lineno,
+                            signature=msig,
+                            summary=mdoc[:120] if mdoc else None,
+                            body=ast.get_source_segment(source, item),
+                        )
+                    )
+
+    # top-level functions
+    for node in ast.iter_child_nodes(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            sig = f"def {node.name}({_args_str(node.args)})"
+            doc = ast.get_docstring(node)
+            symbols.append(
+                Symbol(
+                    name=node.name,
+                    kind="function",
+                    start_line=node.lineno,
+                    end_line=node.end_lineno or node.lineno,
+                    signature=sig,
+                    summary=doc[:120] if doc else None,
+                    body=ast.get_source_segment(source, node),
+                )
+            )
+
+    return symbols
+
+
+def _args_str(args: ast.arguments) -> str:
+    parts: list[str] = []
+    for arg in args.args:
+        parts.append(arg.arg)
+    if args.vararg:
+        parts.append(f"*{args.vararg.arg}")
+    for arg in args.kwonlyargs:
+        parts.append(arg.arg)
+    if args.kwarg:
+        parts.append(f"**{args.kwarg.arg}")
+    return ", ".join(parts)
+
+
+
+_JS_FUNC = re.compile(
+    r"(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(",
+)
+# Require => on the same line to avoid matching non-arrow assignments like:
+#   const result = (a + b)
+_JS_ARROW = re.compile(
+    r"(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|\w+)\s*=>",
+)
+_JS_CLASS = re.compile(r"(?:export\s+)?class\s+(\w+)")
+
+
+def extract_js_symbols(path: Path) -> list[Symbol]:
+    try:
+        lines = path.read_text(errors="replace").splitlines()
+    except OSError:
+        return []
+
+    symbols: list[Symbol] = []
+    brace_depth = 0
+    open_syms: list[tuple[str, str, int]] = []
+
+    for i, line in enumerate(lines, 1):
+        brace_depth += line.count("{") - line.count("}")
+        for pattern, kind in [
+            (_JS_CLASS, "class"),
+            (_JS_FUNC, "function"),
+            (_JS_ARROW, "function"),
+        ]:
+            m = pattern.search(line)
+            if m:
+                open_syms.append((m.group(1), kind, i))
+
+    # close any unclosed syms at end of file
+    end_line = len(lines)
+    for name, kind, start in open_syms:
+        symbols.append(
+            Symbol(
+                name=name,
+                kind=kind,  # type: ignore[arg-type]
+                start_line=start,
+                end_line=end_line,
+                signature=lines[start - 1].strip()[:120],
+                body="\n".join(lines[start - 1 : min(start + 49, end_line)]),
+            )
+        )
+    return symbols
+
+
+def extract_symbols(path: Path, language: str | None) -> list[Symbol]:
+    if language == "python":
+        return extract_python_symbols(path)
+    if language in ("javascript", "typescript"):
+        return extract_js_symbols(path)
+    return []
+
+
+def filter_symbols_by_keywords(symbols: list[Symbol], keywords: set[str]) -> list[Symbol]:
+    """Return symbols whose name or summary matches any keyword."""
+    if not keywords:
+        return symbols
+    result = []
+    for s in symbols:
+        name_lower = s.name.lower()
+        summary_lower = (s.summary or "").lower()
+        if any(kw in name_lower or kw in summary_lower for kw in keywords):
+            result.append(s)
+    return result

agentpack/analysis/tests.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+_TEST_PATTERNS = [
+    # Python
+    lambda stem, parent: f"tests/{parent}/test_{stem}.py",
+    lambda stem, parent: f"test_{stem}.py",
+    lambda stem, parent: f"tests/test_{stem}.py",
+    # JS/TS
+    lambda stem, parent: f"{stem}.test.ts",
+    lambda stem, parent: f"{stem}.spec.ts",
+    lambda stem, parent: f"{stem}.test.js",
+    lambda stem, parent: f"{stem}.spec.js",
+    lambda stem, parent: f"__tests__/{stem}.test.ts",
+    lambda stem, parent: f"__tests__/{stem}.test.js",
+]
+
+
+def find_related_tests(path: str, all_paths: set[str]) -> list[str]:
+    p = Path(path)
+    stem = p.stem
+    parent = p.parent.name
+
+    results: list[str] = []
+    for pattern_fn in _TEST_PATTERNS:
+        candidate = pattern_fn(stem, parent)
+        if candidate in all_paths:
+            results.append(candidate)
+    return results