ravmem-cli 0.1.0__py3-none-any.whl

ravmem/cli/analyze.py

@@ -0,0 +1,484 @@
+"""ravmem analyze — index a repository via the remote server."""
+
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from ravmem.config import DEFAULT_LANGS
+
+# Extensions whose content is never useful for the code-graph pipeline.
+# Oversized text files (> _CLI_MAX_FILE_BYTES) are also skipped.
+_BINARY_SUFFIXES = {
+    # Images
+    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".tiff",
+    # Documents
+    ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
+    # Archives
+    ".zip", ".tar", ".gz", ".bz2", ".xz", ".rar", ".7z",
+    # Compiled / binary
+    ".exe", ".dll", ".so", ".dylib", ".bin", ".o", ".a", ".lib",
+    ".pyc", ".pyo", ".pyd", ".class", ".jar", ".wasm",
+    # Media
+    ".mp3", ".mp4", ".avi", ".mov", ".wav", ".flac", ".ogg",
+    # Fonts
+    ".ttf", ".otf", ".woff", ".woff2",
+    # Data / model files
+    ".pkl", ".joblib", ".npy", ".npz", ".h5", ".hdf5", ".onnx", ".pt", ".pth",
+    # Database
+    ".db", ".sqlite", ".sqlite3",
+}
+
+# Match server default RAVMEM_MAX_FILE_SIZE_MB=5 to avoid uploading files the
+# server would skip anyway.
+_CLI_MAX_FILE_BYTES = 5 * 1024 * 1024
+
+
+def _get_client(remote: str | None):
+    from ravmem.client.api import RavMemClient
+    from ravmem.client.config import load_config
+
+    url = remote
+    api_key = None
+    if not url:
+        cfg = load_config()
+        url = cfg.get("server_url")
+        api_key = cfg.get("api_key")
+    if not url:
+        typer.echo(
+            "Error: No server configured. "
+            "Run `ravmem remote config --url <URL>` or pass --remote <URL>.",
+            err=True,
+        )
+        raise typer.Exit(1)
+    return RavMemClient(base_url=url, api_key=api_key or None)
+
+
+def _print_job_result(job: dict) -> None:
+    status = job.get("status", "unknown")
+    if status == "done":
+        typer.echo(f"✓ Indexing complete (job {job.get('job_id', '?')}).")
+    elif status == "failed":
+        typer.echo(
+            f"✗ Indexing failed (job {job.get('job_id', '?')}): "
+            f"{job.get('failure_reason', 'unknown')}",
+            err=True,
+        )
+        raise typer.Exit(1)
+    else:
+        typer.echo(f"Job {job.get('job_id', '?')} ended with status: {status}")
+
+
+def _analyze_upload(
+    client,
+    project_id: str,
+    repo_path: Path,
+    branch_name: str,
+    parent_branch: str | None,
+    batch_size: int,
+    lang_list: list[str],
+    wait: bool,
+) -> None:
+    """Upload committed files to the server in batches and trigger indexing."""
+    from ravmem.cli.git_local import ls_files, rev_parse, LocalGitError
+
+    typer.echo(
+        "  ℹ  ravmem indexes committed files only. "
+        "Uncommitted changes will not appear.\n"
+        "     Commit your work before running analyze if you want it included."
+    )
+
+    # Collect committed files
+    try:
+        all_rel_paths = ls_files(repo_path)
+    except LocalGitError as exc:
+        typer.echo(f"✗ git ls-files failed: {exc}", err=True)
+        raise typer.Exit(1)
+
+    # Get HEAD commit SHA to record as last_indexed_commit
+    try:
+        to_commit = rev_parse(repo_path)
+    except LocalGitError:
+        to_commit = ""
+
+    # Filter binaries and oversized files
+    to_upload: list[str] = []
+    skipped = 0
+    for rel in all_rel_paths:
+        p = repo_path / rel
+        if p.suffix.lower() in _BINARY_SUFFIXES:
+            skipped += 1
+            continue
+        try:
+            if p.stat().st_size > _CLI_MAX_FILE_BYTES:
+                skipped += 1
+                continue
+        except OSError:
+            skipped += 1
+            continue
+        to_upload.append(rel)
+
+    if not to_upload:
+        typer.echo("No source files found to upload.")
+        return
+
+    total_files = len(to_upload)
+    batches = [to_upload[i : i + batch_size] for i in range(0, total_files, batch_size)]
+    batch_total = len(batches)
+
+    typer.echo(
+        f"  Uploading {total_files} files"
+        + (f" ({skipped} skipped)" if skipped else "")
+        + f" in {batch_total} batch(es)…"
+    )
+
+    job_id: int | None = None
+    for batch_index, batch_paths in enumerate(batches, 1):
+        files: list[dict] = []
+        for rel in batch_paths:
+            p = repo_path / rel
+            try:
+                content = base64.b64encode(p.read_bytes()).decode()
+                files.append({"path": rel, "content": content})
+            except OSError:
+                pass  # unreadable — skip silently
+
+        result = client.upload_batch(
+            project_id=project_id,
+            branch=branch_name,
+            batch_index=batch_index,
+            batch_total=batch_total,
+            files=files,
+            parent_branch=parent_branch,
+            to_commit=to_commit,
+            langs=lang_list if lang_list else None,
+        )
+
+        if batch_total > 1:
+            typer.echo(f"  Batch {batch_index}/{batch_total} sent ({len(files)} files).")
+
+        if "job_id" in result:
+            job_id = result["job_id"]
+
+    if job_id is None:
+        typer.echo("✗ Server did not return a job_id.", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Indexing started on branch '{branch_name}': job_id={job_id}")
+
+    if wait:
+        typer.echo("Waiting for completion…")
+        job = client.poll_job(job_id)
+        _print_job_result(job)
+
+
+def _analyze_incremental_upload(
+    client,
+    project_id: str,
+    repo_path: Path,
+    branch_name: str,
+    parent_branch: str | None,
+    batch_size: int,
+    lang_list: list[str],
+    wait: bool,
+) -> None:
+    """Upload only the files changed since the last index, plus a deletion list."""
+    from ravmem.cli.git_local import diff_name_status, rev_parse, LocalGitError
+
+    # 1. Find out where the server's index is anchored
+    try:
+        status = client.get_branch_status(project_id, branch_name)
+        last_commit = status.get("last_indexed_commit") or ""
+    except Exception:
+        last_commit = ""
+
+    if not last_commit:
+        typer.echo(
+            "  ℹ  No previous index found for this branch — running full upload instead."
+        )
+        _analyze_upload(
+            client=client,
+            project_id=project_id,
+            repo_path=repo_path,
+            branch_name=branch_name,
+            parent_branch=parent_branch,
+            batch_size=batch_size,
+            lang_list=lang_list,
+            wait=wait,
+        )
+        return
+
+    # 2. Diff from the last indexed commit to HEAD
+    try:
+        modified, deleted = diff_name_status(repo_path, last_commit)
+        to_commit = rev_parse(repo_path)
+    except LocalGitError as exc:
+        typer.echo(f"✗ git diff failed: {exc}", err=True)
+        raise typer.Exit(1)
+
+    if not modified and not deleted:
+        typer.echo(
+            f"  ℹ  No changes since last index ({last_commit[:7]}). Nothing to do."
+        )
+        return
+
+    # 3. Filter modified files (same binary/size rules as full upload)
+    to_upload: list[str] = []
+    skipped = 0
+    for rel in modified:
+        p = repo_path / rel
+        if p.suffix.lower() in _BINARY_SUFFIXES:
+            skipped += 1
+            continue
+        try:
+            if p.stat().st_size > _CLI_MAX_FILE_BYTES:
+                skipped += 1
+                continue
+        except OSError:
+            skipped += 1
+            continue
+        to_upload.append(rel)
+
+    typer.echo(
+        f"  Incremental: {len(to_upload)} modified"
+        + (f" ({skipped} skipped)" if skipped else "")
+        + f", {len(deleted)} deleted, base {last_commit[:7]}…"
+    )
+
+    # 4. Build batches — at least one batch even if no modified files (deletions only)
+    batches: list[list[str]] = (
+        [to_upload[i : i + batch_size] for i in range(0, len(to_upload), batch_size)]
+        if to_upload
+        else [[]]
+    )
+    batch_total = len(batches)
+
+    job_id: int | None = None
+    for batch_index, batch_paths in enumerate(batches, 1):
+        files: list[dict] = []
+        for rel in batch_paths:
+            p = repo_path / rel
+            try:
+                content = base64.b64encode(p.read_bytes()).decode()
+                files.append({"path": rel, "content": content})
+            except OSError:
+                pass  # unreadable — skip silently
+
+        result = client.upload_batch(
+            project_id=project_id,
+            branch=branch_name,
+            batch_index=batch_index,
+            batch_total=batch_total,
+            files=files,
+            parent_branch=parent_branch,
+            to_commit=to_commit,
+            from_commit=last_commit,
+            mode="incremental",
+            deleted_paths=deleted if batch_index == 1 else [],
+            langs=lang_list if lang_list else None,
+        )
+
+        if batch_total > 1:
+            typer.echo(f"  Batch {batch_index}/{batch_total} sent ({len(files)} files).")
+
+        if "job_id" in result:
+            job_id = result["job_id"]
+
+    if job_id is None:
+        typer.echo("✗ Server did not return a job_id.", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Incremental indexing started on branch '{branch_name}': job_id={job_id}")
+
+    if wait:
+        typer.echo("Waiting for completion…")
+        job = client.poll_job(job_id)
+        _print_job_result(job)
+
+
+def _analyze_server_side(
+    client,
+    project_id: str,
+    branch_name: str,
+    mode: str,
+    lang_list: list[str],
+    wait: bool,
+    yes: bool,
+    no_auto_sync: bool = False,
+) -> None:
+    """Trigger server-side indexing (mount / clone projects)."""
+    # Pre-flight divergence check
+    try:
+        div = client.get_divergence(project_id, branch_name)
+        ratio = div.get("ratio", 0.0)
+        recommend = div.get("recommend", "full")
+        parent_branch_name = div.get("parent_branch") or ""
+        parent_behind = div.get("parent_behind", False)
+
+        # Cross-branch guard: warn when the parent index is stale (Phase 12).
+        # An incremental index built on a stale baseline may silently miss changes
+        # that landed on the parent after its last index run.
+        if parent_behind and parent_branch_name and not yes:
+            typer.echo(
+                f"\nWarning: parent branch '{parent_branch_name}' index is not current.\n"
+                f"  Re-indexing '{parent_branch_name}' first will give more accurate incremental results."
+            )
+            if not typer.confirm("Proceed anyway?", default=False):
+                raise typer.Exit(0)
+
+        if ratio > 0.60 and recommend == "full" and mode == "auto" and not yes:
+            pct = int(ratio * 100)
+            typer.echo(
+                f"\nWarning: {div.get('changed_files', '?')}/{div.get('total_files', '?')} "
+                f"files changed since last index ({pct}%).\n"
+                "This will trigger a full re-index."
+            )
+            if not typer.confirm("Proceed?", default=False):
+                raise typer.Exit(0)
+        if mode == "auto":
+            mode = recommend
+    except typer.Exit:
+        raise
+    except Exception:
+        pass  # divergence check is best-effort
+
+    result = client.index_branch(
+        project_id, branch_name, mode=mode, langs=lang_list or None,
+        no_auto_sync=no_auto_sync,
+    )
+    job_id = result.get("job_id")
+    typer.echo(f"Indexing started on branch '{branch_name}': job_id={job_id}")
+
+    if wait and job_id is not None:
+        typer.echo("Waiting for completion…")
+        job = client.poll_job(job_id)
+        _print_job_result(job)
+
+
+def analyze_cmd(
+    repo: Path = typer.Argument(Path("."), help="Path to the repository to index"),
+    langs: str = typer.Option(
+        ",".join(DEFAULT_LANGS),
+        "--langs",
+        help="Comma-separated list of languages to index",
+    ),
+    incremental: bool = typer.Option(
+        False, "--incremental", "-i",
+        help="Run incremental update from last indexed commit (upload mode: patches; clone/mount: git-diff)",
+    ),
+    remote: Optional[str] = typer.Option(None, "--remote", help="Server URL (overrides config)"),
+    project: Optional[str] = typer.Option(None, "--project", help="Project ID on the server"),
+    branch: Optional[str] = typer.Option(
+        None, "--branch", help="Branch name (auto-detected from git if omitted)"
+    ),
+    parent_branch: Optional[str] = typer.Option(
+        None, "--parent-branch",
+        help="Parent branch for copy-from-base optimisation (defaults to project default branch)",
+    ),
+    batch_size: int = typer.Option(
+        50, "--batch-size",
+        help="Files per upload batch (upload mode only; default 50)",
+        min=1, max=200,
+    ),
+    default_branch: Optional[str] = typer.Option(
+        None, "--default-branch",
+        help="Set/update the project's default/parent branch",
+    ),
+    wait: bool = typer.Option(False, "--wait", "-w", help="Wait for indexing to complete"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompts (CI/scripts)"),
+    no_auto_sync: bool = typer.Option(
+        False, "--no-auto-sync",
+        help="Skip automatic git fetch before indexing (clone-mode only). "
+             "Use when you know the workspace is already up to date.",
+    ),
+) -> None:
+    """Index a repository — detects upload/clone/mount mode from the project registration."""
+
+    repo_path = repo.resolve()
+
+    client = _get_client(remote)
+    try:
+        if not project:
+            typer.echo(
+                "✗ --project is required. "
+                "Register first with `ravmem remote add`.",
+                err=True,
+            )
+            raise typer.Exit(1)
+
+        # Fetch project info to determine index_mode and default_branch
+        try:
+            project_info = client.get_project(project)
+        except Exception as exc:
+            typer.echo(f"✗ Could not fetch project '{project}': {exc}", err=True)
+            raise typer.Exit(1)
+
+        index_mode = project_info.get("index_mode", "mount")
+        project_default_branch = project_info.get("default_branch", "main")
+
+        # Update default branch if requested
+        if default_branch:
+            client.update_project(project, default_branch=default_branch)
+            typer.echo(f"Default branch updated to '{default_branch}'.")
+            project_default_branch = default_branch
+
+        # Detect branch
+        if branch:
+            branch_name = branch
+        elif index_mode == "upload":
+            from ravmem.cli.git_local import current_branch
+            branch_name = current_branch(repo_path) or project_default_branch
+        else:
+            from ravmem.cli.git_local import current_branch
+            branch_name = current_branch(repo_path) or project_default_branch
+
+        effective_parent = parent_branch or project_default_branch
+        lang_list = [ln.strip() for ln in langs.split(",") if ln.strip()]
+
+        typer.echo(
+            f"Analyzing '{project}' branch '{branch_name}' "
+            f"[mode: {index_mode}]…"
+        )
+
+        if index_mode == "upload":
+            if incremental:
+                _analyze_incremental_upload(
+                    client=client,
+                    project_id=project,
+                    repo_path=repo_path,
+                    branch_name=branch_name,
+                    parent_branch=effective_parent,
+                    batch_size=batch_size,
+                    lang_list=lang_list,
+                    wait=wait,
+                )
+            else:
+                _analyze_upload(
+                    client=client,
+                    project_id=project,
+                    repo_path=repo_path,
+                    branch_name=branch_name,
+                    parent_branch=effective_parent,
+                    batch_size=batch_size,
+                    lang_list=lang_list,
+                    wait=wait,
+                )
+        else:
+            # mount or clone — server-side indexing
+            mode = "incremental" if incremental else "auto"
+            _analyze_server_side(
+                client=client,
+                project_id=project,
+                branch_name=branch_name,
+                mode=mode,
+                lang_list=lang_list,
+                wait=wait,
+                yes=yes,
+                no_auto_sync=no_auto_sync,
+            )
+
+    finally:
+        client.close()