repolens-cli 0.1.0__py3-none-any.whl

repolens/fetcher.py

@@ -0,0 +1,198 @@
+"""GitHub API client — fetches file trees and content without cloning."""
+import base64
+import re
+import time
+from typing import Optional
+
+import requests
+
+from .models import FileNode
+
+GITHUB_API = "https://api.github.com"
+
+SUPPORTED_EXTENSIONS = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".go": "go",
+    ".java": "java",
+    ".rb": "ruby",
+    ".rs": "rust",
+    ".cpp": "cpp",
+    ".c": "c",
+    ".h": "c",
+    ".cs": "csharp",
+    ".php": "php",
+    ".swift": "swift",
+    ".kt": "kotlin",
+    ".scala": "scala",
+    ".md": "markdown",
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+}
+
+
+def parse_github_url(url: str) -> tuple[str, str, Optional[str]]:
+    """Parse GitHub URL into (owner, repo, branch_or_None).
+
+    Handles:
+    - https://github.com/owner/repo
+    - https://github.com/owner/repo/tree/branch
+    - github.com/owner/repo
+    - owner/repo
+    """
+    url = url.strip().rstrip("/")
+    # strip protocol
+    url = re.sub(r"^https?://", "", url)
+    url = re.sub(r"^github\.com/", "", url)
+
+    parts = url.split("/")
+    if len(parts) < 2:
+        raise ValueError(f"Cannot parse GitHub URL: {url!r}. Expected owner/repo format.")
+
+    owner, repo = parts[0], parts[1]
+    repo = repo.removesuffix(".git")
+
+    branch = None
+    if len(parts) >= 4 and parts[2] == "tree":
+        branch = "/".join(parts[3:])
+
+    return owner, repo, branch
+
+
+def _headers(token: Optional[str]) -> dict:
+    h = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
+    if token:
+        h["Authorization"] = f"Bearer {token}"
+    return h
+
+
+def _get(url: str, token: Optional[str], params: dict | None = None) -> dict:
+    resp = requests.get(url, headers=_headers(token), params=params, timeout=30)
+    if resp.status_code == 403 and "rate limit" in resp.text.lower():
+        reset = int(resp.headers.get("X-RateLimit-Reset", time.time() + 60))
+        wait = max(0, reset - int(time.time())) + 1
+        raise RuntimeError(
+            f"GitHub rate limit hit. Resets in {wait}s. Set GITHUB_TOKEN to increase limits."
+        )
+    if resp.status_code == 404:
+        raise ValueError(f"Not found: {url}")
+    resp.raise_for_status()
+    return resp.json()
+
+
+def get_repo_info(owner: str, repo: str, token: Optional[str] = None) -> dict:
+    return _get(f"{GITHUB_API}/repos/{owner}/{repo}", token)
+
+
+def get_default_branch(owner: str, repo: str, token: Optional[str] = None) -> str:
+    info = get_repo_info(owner, repo, token)
+    return info["default_branch"]
+
+
+def fetch_file_tree(
+    owner: str,
+    repo: str,
+    branch: str,
+    token: Optional[str] = None,
+    max_files: int = 500,
+) -> list[FileNode]:
+    """Fetch the full file tree via GitHub Trees API (no cloning)."""
+    # Get commit SHA for the branch
+    branch_data = _get(f"{GITHUB_API}/repos/{owner}/{repo}/branches/{branch}", token)
+    commit_sha = branch_data["commit"]["sha"]
+
+    # Get tree SHA from commit
+    commit_data = _get(f"{GITHUB_API}/repos/{owner}/{repo}/git/commits/{commit_sha}", token)
+    tree_sha = commit_data["tree"]["sha"]
+
+    # Fetch full recursive tree
+    tree_data = _get(
+        f"{GITHUB_API}/repos/{owner}/{repo}/git/trees/{tree_sha}",
+        token,
+        params={"recursive": "1"},
+    )
+
+    if tree_data.get("truncated"):
+        print(f"  Warning: repo tree was truncated (>{max_files} items). Showing first {max_files}.")
+
+    nodes = []
+    for item in tree_data.get("tree", []):
+        if item["type"] != "blob":
+            continue
+        path = item["path"]
+        ext = "." + path.rsplit(".", 1)[-1] if "." in path else ""
+        language = SUPPORTED_EXTENSIONS.get(ext.lower(), "")
+        nodes.append(
+            FileNode(
+                path=path,
+                sha=item["sha"],
+                size=item.get("size", 0),
+                language=language,
+            )
+        )
+        if len(nodes) >= max_files:
+            break
+
+    return nodes
+
+
+def fetch_file_content(
+    owner: str,
+    repo: str,
+    path: str,
+    branch: str,
+    token: Optional[str] = None,
+) -> Optional[str]:
+    """Fetch decoded content of a single file. Returns None if too large or binary."""
+    try:
+        data = _get(
+            f"{GITHUB_API}/repos/{owner}/{repo}/contents/{path}",
+            token,
+            params={"ref": branch},
+        )
+    except (ValueError, requests.HTTPError):
+        return None
+
+    if isinstance(data, list):
+        return None  # It's a directory
+
+    if data.get("encoding") != "base64":
+        return None
+
+    size = data.get("size", 0)
+    if size > 500_000:  # skip files > 500KB
+        return None
+
+    raw = data.get("content", "")
+    try:
+        return base64.b64decode(raw).decode("utf-8", errors="replace")
+    except Exception:
+        return None
+
+
+def fetch_source_files(
+    owner: str,
+    repo: str,
+    branch: str,
+    files: list[FileNode],
+    token: Optional[str] = None,
+    languages: Optional[set[str]] = None,
+    max_fetch: int = 200,
+) -> None:
+    """Fetch content for source files in-place, filtered by language."""
+    source_files = [
+        f for f in files
+        if f.language and f.language not in ("markdown", "json", "yaml", "toml")
+        and (languages is None or f.language in languages)
+    ][:max_fetch]
+
+    print(f"  Fetching content for {len(source_files)} source files...")
+    for i, file_node in enumerate(source_files, 1):
+        if i % 20 == 0:
+            print(f"    {i}/{len(source_files)}...")
+        file_node.content = fetch_file_content(owner, repo, file_node.path, branch, token)

repolens/graph.py

@@ -0,0 +1,126 @@
+"""Build and analyse dependency and call graphs from FileAnalysis data."""
+from __future__ import annotations
+
+from .models import FileAnalysis, FunctionNode, GraphStats
+
+
+def build_graph(analyses: dict[str, FileAnalysis]) -> GraphStats:
+    stats = GraphStats()
+
+    # ── Import graph ──────────────────────────────────────────────────────────
+    in_degree: dict[str, int] = {p: 0 for p in analyses}
+
+    for path, fa in analyses.items():
+        unique_imports = list(dict.fromkeys(fa.resolved_imports))  # deduplicate, preserve order
+        stats.import_edges[path] = unique_imports
+        for dep in unique_imports:
+            if dep in in_degree:
+                in_degree[dep] = in_degree.get(dep, 0) + 1
+            else:
+                in_degree[dep] = 1
+
+    stats.in_degree = in_degree
+
+    # ── Circular dependency detection (iterative DFS) ─────────────────────────
+    stats.circular_deps = _find_cycles(stats.import_edges)
+
+    # ── Entry points (source files nobody imports) ────────────────────────────
+    stats.entry_points = [
+        p for p in analyses if in_degree.get(p, 0) == 0
+    ]
+
+    # ── Hub files (most imported) ─────────────────────────────────────────────
+    stats.hub_files = sorted(
+        [(p, in_degree.get(p, 0)) for p in analyses],
+        key=lambda x: x[1],
+        reverse=True,
+    )[:10]
+
+    # ── Function index & caller resolution ────────────────────────────────────
+    # Build name -> list[function_id] index for resolving calls
+    name_index: dict[str, list[str]] = {}
+    for path, fa in analyses.items():
+        for fn in fa.functions:
+            fid = f"{path}::{fn.name}"
+            fn_copy = FunctionNode(
+                name=fn.name,
+                file_path=fn.file_path,
+                line_start=fn.line_start,
+                line_end=fn.line_end,
+                calls=list(fn.calls),
+            )
+            stats.functions[fid] = fn_copy
+            name_index.setdefault(fn.name, []).append(fid)
+
+    # Resolve calls to function IDs and back-populate callers
+    for fid, fn in stats.functions.items():
+        resolved_calls: list[str] = []
+        for call in fn.calls:
+            base_name = call.split(".")[-1]  # handle obj.method style
+            for candidate in name_index.get(base_name, []):
+                resolved_calls.append(candidate)
+                stats.functions[candidate].callers.append(fid)
+        fn.calls = list(dict.fromkeys(resolved_calls))
+
+    return stats
+
+
+def _find_cycles(edges: dict[str, list[str]]) -> list[list[str]]:
+    """Detect all simple cycles using iterative DFS (Johnson-lite)."""
+    WHITE, GRAY, BLACK = 0, 1, 2
+    color: dict[str, int] = {n: WHITE for n in edges}
+    # include all targets too
+    for deps in edges.values():
+        for d in deps:
+            color.setdefault(d, WHITE)
+
+    cycles: list[list[str]] = []
+    parent: dict[str, str | None] = {}
+
+    for start in list(color):
+        if color[start] != WHITE:
+            continue
+        stack = [(start, iter(edges.get(start, [])))]
+        color[start] = GRAY
+        path = [start]
+        parent[start] = None
+
+        while stack:
+            node, children = stack[-1]
+            try:
+                child = next(children)
+                if color.get(child, WHITE) == GRAY:
+                    # Found a back-edge → extract cycle
+                    idx = path.index(child)
+                    cycle = path[idx:]
+                    # Deduplicate: only add if we haven't seen this cycle (by sorted set)
+                    cycle_key = frozenset(cycle)
+                    if not any(frozenset(c) == cycle_key for c in cycles):
+                        cycles.append(cycle)
+                elif color.get(child, WHITE) == WHITE:
+                    color[child] = GRAY
+                    parent[child] = node
+                    path.append(child)
+                    stack.append((child, iter(edges.get(child, []))))
+            except StopIteration:
+                color[node] = BLACK
+                stack.pop()
+                if path and path[-1] == node:
+                    path.pop()
+
+    return cycles
+
+
+def importers_of(path: str, stats: GraphStats) -> list[str]:
+    """Return list of files that import *path*."""
+    return [src for src, deps in stats.import_edges.items() if path in deps]
+
+
+def callers_of(function_id: str, stats: GraphStats) -> list[str]:
+    fn = stats.functions.get(function_id)
+    return fn.callers if fn else []
+
+
+def callees_of(function_id: str, stats: GraphStats) -> list[str]:
+    fn = stats.functions.get(function_id)
+    return fn.calls if fn else []

repolens/models.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class FileNode:
+    path: str           # repo-relative path
+    size: int = 0
+    language: str = ""
+    content: Optional[str] = None
+
+
+@dataclass
+class FunctionNode:
+    name: str
+    file_path: str
+    line_start: int
+    line_end: int
+    calls: list[str] = field(default_factory=list)   # raw call names found in body
+    callers: list[str] = field(default_factory=list)  # populated by graph builder
+    docstring: Optional[str] = None
+
+
+@dataclass
+class FileAnalysis:
+    path: str
+    language: str
+    raw_imports: list[str] = field(default_factory=list)
+    resolved_imports: list[str] = field(default_factory=list)  # repo-relative paths
+    functions: list[FunctionNode] = field(default_factory=list)
+    classes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class GraphStats:
+    # file_path -> list of files it imports (within repo)
+    import_edges: dict[str, list[str]] = field(default_factory=dict)
+    # file_path -> in-degree (how many files import it)
+    in_degree: dict[str, int] = field(default_factory=dict)
+    circular_deps: list[list[str]] = field(default_factory=list)
+    entry_points: list[str] = field(default_factory=list)   # nothing imports these
+    hub_files: list[tuple[str, int]] = field(default_factory=list)  # (path, in_degree) top-10
+    # function_id "file::name" -> FunctionNode
+    functions: dict[str, FunctionNode] = field(default_factory=dict)
+
+
+@dataclass
+class RepoAnalysis:
+    root: str          # absolute path to scanned directory
+    files: list[FileNode] = field(default_factory=list)
+    file_analyses: dict[str, FileAnalysis] = field(default_factory=dict)
+    stats: GraphStats = field(default_factory=GraphStats)

repolens/scanner.py

@@ -0,0 +1,69 @@
+import os
+from pathlib import Path
+
+from .models import FileNode
+
+SUPPORTED_EXTENSIONS: dict[str, str] = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+    ".rb": "ruby",
+    ".cpp": "cpp",
+    ".c": "c",
+    ".h": "c",
+    ".cs": "csharp",
+    ".php": "php",
+    ".swift": "swift",
+    ".kt": "kotlin",
+    ".scala": "scala",
+    ".md": "markdown",
+}
+
+SKIP_DIRS = {
+    ".git", ".hg", ".svn",
+    "node_modules", "__pycache__", ".mypy_cache", ".ruff_cache",
+    ".venv", "venv", "env", ".env",
+    "dist", "build", "out", "target",
+    ".idea", ".vscode",
+    "vendor", "third_party",
+}
+
+MAX_FILE_SIZE = 500_000  # 500 KB
+
+
+def scan(root: str, max_files: int = 2000) -> list[FileNode]:
+    """Walk *root* and return FileNode list for all source files."""
+    root_path = Path(root).resolve()
+    nodes: list[FileNode] = []
+
+    for dirpath, dirnames, filenames in os.walk(root_path):
+        # Prune ignored directories in-place
+        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".")]
+
+        for filename in filenames:
+            if len(nodes) >= max_files:
+                break
+            full_path = Path(dirpath) / filename
+            suffix = full_path.suffix.lower()
+            language = SUPPORTED_EXTENSIONS.get(suffix)
+            if not language:
+                continue
+
+            rel_path = full_path.relative_to(root_path).as_posix()
+            size = full_path.stat().st_size
+
+            content: str | None = None
+            if size <= MAX_FILE_SIZE:
+                try:
+                    content = full_path.read_text(encoding="utf-8", errors="replace")
+                except OSError:
+                    pass
+
+            nodes.append(FileNode(path=rel_path, size=size, language=language, content=content))
+
+    return sorted(nodes, key=lambda n: n.path)