polyglotminer 1.0.1__py3-none-any.whl

polyglotminer/__init__.py

@@ -0,0 +1,3 @@
+"""PolyglotMiner - Git Repository Analysis & Cross-Language Migration Research Tool."""
+
+__version__ = "1.0.1"

polyglotminer/cache.py

@@ -0,0 +1,246 @@
+"""Persistent Repository Cache Pool for rapid Git history operations.
+
+Maintains local clones in `.cache/repos/` so repositories are cloned once
+using standard git CLI and updated via sub-second `git fetch`.
+"""
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+from typing import Optional, Tuple, Union
+
+from polyglotminer.scanner import (
+    format_git_auth_header,
+    get_github_auth_token,
+    invalidate_github_token,
+    is_auth_failure_error,
+    resolve_remote_url,
+)
+
+
+def get_cache_root() -> Path:
+    """Return the absolute path to workspace repository cache pool (.cache/repos)."""
+    base = Path(__file__).resolve().parent.parent / ".cache" / "repos"
+    base.mkdir(parents=True, exist_ok=True)
+    return base
+
+
+def get_cached_repo_dir(remote_target: Union[str, Path]) -> Tuple[Path, str, str]:
+    """Return (local_cache_path, clone_url, repo_slug)."""
+    clone_url, default_name = resolve_remote_url(remote_target)
+    slug = default_name.replace("/", "_").replace("\\", "_")
+    short_name = default_name.split("/")[-1] if "/" in default_name else default_name
+
+    root = get_cache_root()
+    # Prioritize the unique owner-qualified slug to prevent collisions
+    if (root / slug / ".git").exists():
+        return root / slug, clone_url, default_name
+    if (root / short_name / ".git").exists():
+        return root / short_name, clone_url, default_name
+
+    target_dir = root / slug
+    return target_dir, clone_url, default_name
+
+
+def resolve_repo_path(
+    target: Union[str, Path],
+    reports_dir: Optional[Union[str, Path]] = None,
+) -> Optional[Path]:
+    """Resolve a repository target string or Path to an existing local git repository Path if available."""
+    if not target:
+        return None
+    p = Path(target).resolve()
+    if p.exists() and (p / ".git").exists():
+        return p
+    cached_dir, _, _ = get_cached_repo_dir(str(target))
+    if cached_dir.exists() and (cached_dir / ".git").exists():
+        return cached_dir
+    root = get_cache_root()
+    slug = str(target).replace("/", "_").replace("\\", "_")
+    short_name = str(target).split("/")[-1] if "/" in str(target) else str(target)
+    for candidate in (root / slug, root / short_name, root / str(target)):
+        if candidate.exists() and (candidate / ".git").exists():
+            return candidate
+    return None
+
+
+def _force_rmtree(path: Path) -> None:
+    """Force recursive directory removal on Windows handling read-only git files."""
+    import stat
+    if not path.exists():
+        return
+
+    def _handle_remove_readonly(func, p, exc=None):
+        try:
+            os.chmod(p, stat.S_IWRITE)
+            func(p)
+        except Exception:
+            pass
+
+    try:
+        if sys.version_info >= (3, 12):
+            shutil.rmtree(path, onexc=_handle_remove_readonly)
+        else:
+            shutil.rmtree(path, onerror=_handle_remove_readonly)
+    except Exception:
+        pass
+
+    if path.exists() and sys.platform == "win32":
+        try:
+            subprocess.run(["cmd", "/c", "rmdir", "/s", "/q", str(path)], capture_output=True, timeout=5)
+        except Exception:
+            pass
+
+
+def get_or_clone_repository(
+    remote_target: Union[str, Path],
+    blobless: bool = True,
+    fetch: bool = False,
+    progress_callback: Optional[callable] = None,
+) -> Path:
+    """Obtain a cached repository Path, cloning if missing or fetching if present."""
+    env = dict(os.environ)
+    env["GIT_TERMINAL_PROMPT"] = "0"
+    env["GIT_ASKPASS"] = ""
+
+    cached_dir, clone_url, repo_slug = get_cached_repo_dir(remote_target)
+
+    # Clean up any partial/broken directories
+    git_dir = cached_dir / ".git"
+    if cached_dir.exists():
+        if git_dir.exists():
+            test_res = subprocess.run(
+                ["git", "-C", str(cached_dir), "rev-parse", "HEAD"],
+                capture_output=True,
+                env=env,
+                timeout=5,
+            )
+            if test_res.returncode == 0:
+                # Valid existing clone
+                if not blobless:
+                    # Check if existing clone is blobless (missing blobs)
+                    check_filter = subprocess.run(
+                        ["git", "-C", str(cached_dir), "config", "--get", "remote.origin.partialclonefilter"],
+                        capture_output=True,
+                        text=True,
+                        env=env,
+                    )
+                    if "blob:none" in check_filter.stdout:
+                        # Re-clone cleanly with full objects so cat-file line counting is 100% local and instantaneous
+                        _force_rmtree(cached_dir)
+
+                if cached_dir.exists():
+                    if fetch:
+                        if progress_callback:
+                            progress_callback(1, 2, f"Updating cached repository {repo_slug} (git fetch)...")
+                        try:
+                            subprocess.run(
+                                ["git", "fetch", "--all"],
+                                cwd=str(cached_dir),
+                                env=env,
+                                stdin=subprocess.DEVNULL,
+                                capture_output=True,
+                                check=False,
+                                timeout=30,
+                            )
+                        except Exception:
+                            pass
+                    return cached_dir
+
+        # Corrupted / invalid directory: remove cleanly
+        _force_rmtree(cached_dir)
+
+    # Repository not in cache: perform robust initial clone via git CLI
+    if progress_callback:
+        progress_callback(1, 2, f"Cloning {repo_slug} into cache pool...")
+
+    cached_dir.parent.mkdir(parents=True, exist_ok=True)
+    token = get_github_auth_token()
+
+    def _execute_clone(auth_token: Optional[str]) -> subprocess.CompletedProcess:
+        cmd = [
+            "git",
+            "-c", "credential.helper=",
+            "-c", "core.askPass=",
+        ]
+        if auth_token:
+            cmd.extend(["-c", f"http.extraHeader={format_git_auth_header(auth_token)}"])
+        cmd.extend(["clone", "--no-checkout"])
+
+        if blobless:
+            cmd.append("--filter=blob:none")
+
+        cmd.extend([
+            clone_url,
+            str(cached_dir),
+        ])
+
+        return subprocess.run(
+            cmd,
+            env=env,
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+
+    result = _execute_clone(token)
+
+    # Detect if clone failed due to authentication issues (e.g. expired/revoked token)
+    if result.returncode != 0 and token:
+        err_msg = result.stderr or result.stdout or ""
+        if is_auth_failure_error(err_msg):
+            invalidate_github_token()
+            if cached_dir.exists():
+                _force_rmtree(cached_dir)
+            if progress_callback:
+                progress_callback(1, 2, f"GitHub token rejected for {repo_slug}. Retrying unauthenticated clone...")
+            result = _execute_clone(None)
+
+    if result.returncode != 0:
+        if cached_dir.exists():
+            _force_rmtree(cached_dir)
+        clean_url = clone_url.split("@")[-1] if "@" in clone_url else clone_url
+        raise RuntimeError(f"Git clone failed for {clean_url}: {result.stderr or result.stdout}")
+
+    # Verify that the cloned repository is valid
+    test_res = subprocess.run(
+        ["git", "-C", str(cached_dir), "rev-parse", "HEAD"],
+        capture_output=True,
+        env=env,
+        timeout=5,
+    )
+    if test_res.returncode != 0:
+        _force_rmtree(cached_dir)
+        raise RuntimeError(f"Cloned repository {clone_url} has no valid HEAD ref.")
+
+    return cached_dir
+
+
+def clear_repo_cache(target_slug: Optional[Union[str, Path]] = None) -> int:
+    """Clear cached repositories. If target_slug is None, clears all."""
+    root = get_cache_root()
+    if not root.exists():
+        return 0
+
+    count = 0
+    if target_slug:
+        cached_dir, _, _ = get_cached_repo_dir(str(target_slug))
+        if cached_dir.exists():
+            _force_rmtree(cached_dir)
+            count = 1
+        else:
+            slug = str(target_slug).replace("/", "_").replace("\\", "_")
+            short = str(target_slug).split("/")[-1]
+            for p in [root / slug, root / short, root / str(target_slug)]:
+                if p.exists():
+                    _force_rmtree(p)
+                    count += 1
+    else:
+        for child in root.iterdir():
+            if child.is_dir():
+                _force_rmtree(child)
+                count += 1
+    return count