sql-code-graph 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

sqlcg/cli/commands/index.py

@@ -1,9 +1,18 @@
 """Index command for scanning and indexing SQL files."""
 
+import os
 from pathlib import Path
 
 import typer
 from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeRemainingColumn,
+)
 
 from sqlcg.core.config import get_backend, get_db_path, get_dialect
 from sqlcg.indexer.indexer import Indexer
@@ -20,26 +29,53 @@ def index_cmd(  # noqa: B008
         None, "--dbt-manifest", help="Path to dbt manifest"
     ),
     timeout_per_file: int = typer.Option(  # noqa: B008
-        30, "--timeout-per-file", help="Timeout per file in seconds"
+        5, "--timeout-per-file", help="Timeout per file in seconds"
     ),
-    no_ddl: bool = typer.Option(  # noqa: B008
-        False, "--no-ddl", help="Skip DDL statements (not yet fully implemented)"
+    buffer_pool_size: int = typer.Option(  # noqa: B008
+        0,
+        "--buffer-pool-size",
+        help="KuzuDB buffer pool size in MB (0 = default). "
+        "Set to 256-512 on memory-constrained machines.",
+    ),
+    batch_size: int = typer.Option(  # noqa: B008
+        50,
+        "--batch-size",
+        help=(
+            "Files per KuzuDB transaction in the upsert pass. "
+            "Default 50 balances commit-overhead reduction (vs. legacy per-file commits) "
+            "against per-batch memory cost. Lower values are safer for memory-constrained "
+            "machines; higher values give marginal speedup at the cost of larger working sets. "
+            "Set to 1 to reproduce legacy per-file commit behaviour."
+        ),
     ),
-    schema_from_info_schema: str | None = typer.Option(  # noqa: B008
-        None, "--schema-from-info-schema", hidden=True, help="(Not yet implemented)"
+    no_ddl: bool = typer.Option(  # noqa: B008
+        False, "--no-ddl", help="Skip table-node upserts for DDL-only files"
     ),
     quiet: bool = typer.Option(  # noqa: B008
         False, "--quiet", "-q", help="Suppress summary console output"
     ),
+    debug: bool = typer.Option(  # noqa: B008
+        False, "--debug", help="Show detailed log output during indexing"
+    ),
+    profile: bool = typer.Option(  # noqa: B008
+        False, "--profile/--no-profile", help="Emit per-stage timing after indexing"
+    ),
 ) -> None:
-    """Index SQL files in a directory."""
-    if schema_from_info_schema:
-        console.print("[red]--schema-from-info-schema is not yet implemented (v2)[/red]")
-        raise typer.Exit(1)
+    """Index SQL files in a directory.
 
-    # TODO: wire no_ddl through to the indexer once it supports the parameter
-    if no_ddl:
-        console.print("[yellow]Note: --no-ddl is not yet fully implemented[/yellow]")
+    Schema aliases (staging schema → canonical schema) can be configured in
+    .sqlcg.toml under sqlcg.schema_aliases, e.g. da_tmp = "da".
+    """
+
+    import logging
+
+    level = logging.DEBUG if debug else logging.CRITICAL
+    logging.getLogger("sqlcg").setLevel(level)
+    logging.getLogger("sqlglot").setLevel(level)
+
+    # Set buffer pool size via env var if specified
+    if buffer_pool_size > 0:
+        os.environ["SQLCG_BUFFER_POOL_MB"] = str(buffer_pool_size)
 
     # Resolve dialect: 'auto' reads from .sqlcg.toml, otherwise use provided value
     if dialect == "auto":
@@ -51,8 +87,16 @@ def index_cmd(  # noqa: B008
     with get_backend() as backend:
         backend.init_schema()
 
-        # Create Repo node for this repository
-        from sqlcg.core.schema import NodeLabel
+        # Check schema version — must match current build
+        from sqlcg.core.schema import SCHEMA_VERSION, NodeLabel
+
+        stored = backend.get_schema_version()
+        if stored != SCHEMA_VERSION:
+            console.print(
+                f"[red]Database schema is v{stored}; this build requires v{SCHEMA_VERSION}. "
+                "Run 'sqlcg db reset && sqlcg db init && sqlcg index <path>' to re-index.[/red]"
+            )
+            raise typer.Exit(1)
 
         abs_path = str(path.resolve())
         backend.upsert_node(
@@ -66,7 +110,32 @@ def index_cmd(  # noqa: B008
 
         # Index the repository
         indexer = Indexer()
-        summary = indexer.index_repo(path, dialect, backend, dbt_manifest, timeout_per_file)
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TimeRemainingColumn(),
+            console=console,
+            redirect_stderr=True,
+        ) as progress:
+            task = progress.add_task("Parsing", total=None)
+
+            def _progress_callback(n: int, total_n: int) -> None:
+                progress.update(task, completed=n, total=total_n)
+
+            summary = indexer.index_repo(
+                path,
+                dialect,
+                backend,
+                dbt_manifest,
+                timeout_per_file,
+                progress_callback=_progress_callback,
+                no_ddl=no_ddl,
+                batch_size=batch_size,
+                profile=profile,
+            )
 
         # Connect files to repo
         from sqlcg.core.schema import RelType
@@ -85,13 +154,49 @@ def index_cmd(  # noqa: B008
 
         # Print summary unless --quiet is specified
         if not quiet:
+            quality = summary.get("quality", {})
+            err = summary.get("error_summary", {})
+            n_full = quality.get("full", 0)
+            n_partial = quality.get("table_only", 0)
+            n_ddl = quality.get("scripting_fallback", 0)
+            n_failed = quality.get("failed", 0)
+            n_timeout = err.get("timeout", 0)
+
+            quality_parts = []
+            if n_full:
+                quality_parts.append(f"[green]{n_full} with column lineage[/green]")
+            if n_partial:
+                quality_parts.append(f"[yellow]{n_partial} table-only[/yellow]")
+            if n_ddl:
+                quality_parts.append(f"{n_ddl} DDL-only")
+            if n_timeout:
+                quality_parts.append(f"[red]{n_timeout} timed out[/red]")
+            if n_failed:
+                quality_parts.append(f"[red]{n_failed} failed[/red]")
+
             console.print(
                 f"[green]Indexed[/green] {summary['files_parsed']} files — "
-                f"{summary['tables_found']} tables, {summary['lineage_edges_created']} edges, "
-                f"{summary['parse_errors']} errors"
+                f"{summary['tables_found']} tables, {summary['lineage_edges_created']} edges"
             )
+            if quality_parts:
+                console.print("  " + " · ".join(quality_parts))
             if summary.get("lineage_edges_created", 0) == 0:
                 console.print(
                     "[yellow]Warning: 0 lineage edges extracted — column lineage "
                     "unavailable.[/yellow]"
                 )
+
+        if prof := summary.get("profile"):
+            console.print("\n[bold]Profile[/bold]")
+            console.print(
+                f"  pass1 parse:    {prof['pass1_parse_s']:.2f}s\n"
+                f"  pass2 resolve:  {prof['pass2_resolve_s']:.2f}s\n"
+                f"  upsert:         {prof['upsert_s']:.2f}s\n"
+                f"  star expand:    {prof['star_expand_s']:.2f}s\n"
+                f"  total:          {prof['total_s']:.2f}s\n"
+                f"  ms/file:        {prof['ms_per_file']:.1f}ms  ({prof['files']} files)"
+            )
+            if prof["slowest_files"]:
+                console.print("  [bold]Slowest files[/bold]")
+                for file_path, ms in prof["slowest_files"]:
+                    console.print(f"    {ms:8.1f}ms  {file_path}")

sqlcg/cli/commands/install.py

@@ -3,6 +3,7 @@
 import json
 import os
 import shutil
+import sys
 from pathlib import Path
 
 import typer
@@ -16,12 +17,34 @@ _SERVER_KEY = "sql-code-graph"
 
 def install_cmd(
     dry_run: bool = typer.Option(False, "--dry-run", help="Print config without writing"),
+    scope: str | None = typer.Option(  # noqa: B008
+        None,
+        "--scope",
+        help="Install skill location: 'project' (under --repo) or 'global' (~/.claude/skills/).",
+    ),
+    repo: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--repo",
+        help="Repository root for --scope project (default: current directory).",
+    ),
 ) -> None:
-    """Register sqlcg as an MCP server in Claude Code (~/.claude/settings.json)."""
-    if shutil.which("uvx"):
-        entry: dict = {"command": "uvx", "args": ["sql-code-graph", "mcp", "start"]}
+    """Register sqlcg as an MCP server in Claude Code (~/.claude/settings.json).
+
+    Also provisions a Claude skill file (SKILL.md) at the chosen location.
+    Pass --scope project or --scope global to specify where the skill is written.
+    On a TTY without --scope, an interactive prompt asks for the location.
+    On a non-TTY (CI, scripts) without --scope, the command exits with an error.
+    """
+    # Resolve --scope: explicit flag, TTY prompt, or non-TTY error
+    resolved_scope = _resolve_scope(scope)
+
+    if shutil.which("sqlcg"):
+        entry: dict = {"command": "sqlcg", "args": ["mcp", "start"]}
+    elif shutil.which("uvx"):
+        entry = {"command": "uvx", "args": ["sql-code-graph", "mcp", "start"]}
     else:
-        entry = {"command": "sqlcg", "args": ["mcp", "start"]}
+        console.print("[red]Error:[/red] Neither 'sqlcg' nor 'uvx' found on PATH.")
+        raise typer.Exit(1)
 
     settings_path = _SETTINGS_PATH
     if settings_path.exists():
@@ -38,18 +61,31 @@ def install_cmd(
 
     mcp_servers: dict = settings.setdefault("mcpServers", {})
 
-    if mcp_servers.get(_SERVER_KEY) == entry:
+    existing_entry = mcp_servers.get(_SERVER_KEY)
+    if existing_entry == entry:
         cmd_str = f"{entry['command']} {' '.join(entry['args'])}"
+        console.print(f"[green]Already configured:[/green] {_SERVER_KEY} → {cmd_str}")
+        # Still provision the skill even when MCP entry already exists
+        _provision_skill(resolved_scope, repo, dry_run)
+        return
+
+    # Print upgrade notice if switching from uvx to sqlcg
+    if (
+        existing_entry
+        and existing_entry.get("command") == "uvx"
+        and entry.get("command") == "sqlcg"
+    ):
         console.print(
-            f"[green]Already configured:[/green] {_SERVER_KEY} → {cmd_str}"
+            "[blue]Updating[/blue] MCP entry from [dim]uvx[/dim] to local "
+            "[green]sqlcg[/green] binary (faster startup). Writing…"
         )
-        return
 
     mcp_servers[_SERVER_KEY] = entry
 
     if dry_run is True:
         console.print("[dim]--dry-run: would write:[/dim]")
         console.print_json(json.dumps(settings, indent=2))
+        _provision_skill(resolved_scope, repo, dry_run)
         return
 
     try:
@@ -65,10 +101,113 @@ def install_cmd(
     console.print(f"[dim]Written to {settings_path}[/dim]")
 
     # Note about cold cache if uvx was chosen
-    if entry['command'] == 'uvx':
+    if entry.get("command") == "uvx":
         console.print(
             "[yellow]Note:[/yellow] First startup downloads dependencies (~30s). "
             "Subsequent restarts use cache (~1s)."
         )
 
     console.print("\nRestart Claude Code to pick up the new MCP server.")
+
+    # Provision the skill file
+    _provision_skill(resolved_scope, repo, dry_run)
+
+
+def _resolve_scope(scope: str | None) -> str:
+    """Resolve the install scope from the --scope flag, TTY prompt, or error.
+
+    - If scope is provided and valid, return it.
+    - If scope is None and stdin is a TTY, prompt interactively.
+    - If scope is None and stdin is not a TTY, exit with a clear error.
+
+    Args:
+        scope: Value of the --scope CLI option (None if omitted).
+
+    Returns:
+        "project" or "global".
+
+    Raises:
+        typer.Exit: if scope is invalid or cannot be determined.
+    """
+    valid = {"project", "global"}
+
+    if scope is not None:
+        if scope not in valid:
+            console.print(
+                f"[red]Error:[/red] --scope must be 'project' or 'global', got: {scope!r}"
+            )
+            raise typer.Exit(1)
+        return scope
+
+    # scope is None — check if stdin is a TTY
+    if sys.stdin.isatty():
+        choice = typer.prompt(
+            "Install skill location",
+            default="project",
+            prompt_suffix=" [project/global]: ",
+            show_default=False,
+        )
+        if choice not in valid:
+            console.print(
+                f"[red]Error:[/red] Invalid choice {choice!r}. Must be 'project' or 'global'."
+            )
+            raise typer.Exit(1)
+        return choice
+
+    # Non-TTY without --scope — error, do not guess
+    console.print(
+        "[red]Error:[/red] --scope is required in non-interactive mode. "
+        "Pass --scope project or --scope global."
+    )
+    raise typer.Exit(1)
+
+
+def _provision_skill(scope: str, repo: Path | None, dry_run: bool) -> None:
+    """Write (or describe) the sqlcg SKILL.md file at the chosen location.
+
+    Uses the .tmp + os.replace atomic write idiom. Idempotent: overwrites
+    the sqlcg-owned SKILL.md on each install. If a SKILL.md exists at the
+    target path without the sqlcg frontmatter marker ('name: sqlcg'), warns
+    and skips to avoid clobbering a foreign file.
+
+    Args:
+        scope: "project" or "global".
+        repo: Repo root for project scope (defaults to Path.cwd()).
+        dry_run: If True, print a note but write nothing.
+    """
+    from sqlcg import __version__
+    from sqlcg.server.skill import render_skill
+
+    # Resolve scope root
+    if scope == "global":
+        scope_root = Path.home()
+    else:
+        scope_root = repo if repo is not None else Path.cwd()
+
+    skill_dir = scope_root / ".claude" / "skills" / "sqlcg"
+    skill_path = skill_dir / "SKILL.md"
+
+    if dry_run:
+        console.print(f"[dim]--dry-run: would write skill to {skill_path}[/dim]")
+        return
+
+    # Foreign-file guard: if SKILL.md exists but does not carry sqlcg frontmatter, warn+skip
+    if skill_path.exists():
+        existing = skill_path.read_text()
+        if "name: sqlcg" not in existing:
+            console.print(
+                f"[yellow]Warning:[/yellow] {skill_path} exists but was not created by sqlcg "
+                "(missing 'name: sqlcg' in frontmatter). Skipping to avoid overwriting."
+            )
+            return
+
+    skill_content = render_skill(__version__)
+
+    try:
+        skill_dir.mkdir(parents=True, exist_ok=True)
+        tmp = skill_path.with_suffix(".tmp")
+        tmp.write_text(skill_content)
+        os.replace(tmp, skill_path)
+        console.print(f"[green]Skill written:[/green] {skill_path}")
+    except (OSError, TypeError, AttributeError) as exc:
+        console.print(f"[yellow]Warning:[/yellow] Failed to write skill file: {exc}")

sqlcg/cli/commands/mcp.py

@@ -53,3 +53,15 @@ def mcp_start() -> None:
     from sqlcg.server.server import main as server_main
 
     server_main()
+
+
+@app.command("best-practices")
+def mcp_best_practices() -> None:
+    """Print MCP tool best-practices (the fact/heuristic boundary).
+
+    Same guidance as the bundled Claude skill — useful for humans or agents
+    that have not installed the skill.
+    """
+    from sqlcg.server.skill import render_body
+
+    typer.echo(render_body())

sqlcg/cli/commands/reindex.py

@@ -0,0 +1,170 @@
+"""Reindex command — incremental resync from a git delta.
+
+Two modes:
+  sqlcg reindex --from <sha> --to <sha> <root>
+      Explicit SHAs: call resync_changed(root, from, to, ...).
+  sqlcg reindex <root>
+      Standalone "catch up": read last-indexed SHA from graph metadata,
+      diff against current HEAD. If no stored SHA, do a full index_repo.
+
+Gates on schema version like the watch command (same reset+reinit message).
+"""
+
+import subprocess
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+console = Console()
+
+
+def reindex_cmd(  # noqa: B008
+    path: Path = typer.Argument(..., help="Repository root directory to resync"),  # noqa: B008
+    from_sha: str | None = typer.Option(  # noqa: B008
+        None, "--from", help="Base git SHA (previously-indexed state)"
+    ),
+    to_sha: str | None = typer.Option(  # noqa: B008
+        None, "--to", help="Target git SHA (defaults to HEAD when --from is given)"
+    ),
+    dialect: str | None = typer.Option(  # noqa: B008
+        None, "--dialect", "-d", help="SQL dialect (or 'auto' to read from .sqlcg.toml)"
+    ),
+    quiet: bool = typer.Option(  # noqa: B008
+        False, "--quiet", "-q", help="Suppress summary output"
+    ),
+    batch_size: int = typer.Option(  # noqa: B008
+        50,
+        "--batch-size",
+        help="Files per KuzuDB transaction (same default as index command)",
+    ),
+    timeout_per_file: int = typer.Option(  # noqa: B008
+        5,
+        "--timeout-per-file",
+        help="Per-file parse timeout in seconds",
+    ),
+) -> None:
+    """Incrementally resync the graph after a git branch change or pull.
+
+    When --from and --to are given (e.g. from the post-checkout hook), only the
+    files that changed between those two SHAs are re-parsed, plus the cross-file
+    pass-2 closure (files that SELECT FROM tables defined in changed files).
+
+    Without --from/--to, reads the last-indexed SHA from the database and diffs it
+    against the current HEAD. If no stored SHA is found, falls back to a full index.
+
+    Exits with an error if the database schema version does not match the current
+    build — run 'sqlcg db reset && sqlcg db init && sqlcg index <path>' to re-init.
+    """
+    from sqlcg.core.config import get_backend, get_db_path, get_dialect
+    from sqlcg.core.schema import SCHEMA_VERSION
+    from sqlcg.indexer.indexer import Indexer
+
+    # Resolve dialect
+    if dialect == "auto":
+        dialect = get_dialect(path)
+
+    db_path = get_db_path()
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with get_backend() as backend:
+        backend.init_schema()
+
+        # Schema-version gate (mirrors watch.py)
+        stored_version = backend.get_schema_version()
+        if stored_version != SCHEMA_VERSION:
+            console.print(
+                f"[red]Database schema is v{stored_version}; "
+                f"this build requires v{SCHEMA_VERSION}. "
+                "Run 'sqlcg db reset && sqlcg db init && sqlcg index <path>' "
+                "to re-initialize.[/red]"
+            )
+            raise typer.Exit(1)
+
+        indexer = Indexer()
+
+        # ---- Determine mode -------------------------------------------------------
+        if from_sha is not None:
+            # Explicit-SHA mode
+            effective_to = to_sha or _get_head(path)
+            if not quiet:
+                console.print(
+                    f"Resyncing [cyan]{path}[/cyan] [dim]{from_sha[:8]}..{effective_to[:8]}[/dim]"
+                )
+            summary = indexer.resync_changed(
+                path,
+                from_sha,
+                effective_to,
+                backend,
+                dialect,
+                batch_size=batch_size,
+                timeout_per_file=timeout_per_file,
+            )
+        else:
+            # Standalone mode: use stored SHA -> HEAD
+            old = backend.get_indexed_sha()
+            if old is None:
+                if not quiet:
+                    console.print(
+                        "[yellow]No stored index SHA found — performing full index.[/yellow]"
+                    )
+                indexer.index_repo(
+                    path,
+                    dialect,
+                    backend,
+                    batch_size=batch_size,
+                    timeout_per_file=timeout_per_file,
+                )
+                if not quiet:
+                    console.print("[green]Full index complete.[/green]")
+                return
+            new = _get_head(path)
+            if not quiet:
+                console.print(f"Resyncing [cyan]{path}[/cyan] [dim]{old[:8]}..{new[:8]}[/dim]")
+            summary = indexer.resync_changed(
+                path,
+                old,
+                new,
+                backend,
+                dialect,
+                batch_size=batch_size,
+                timeout_per_file=timeout_per_file,
+            )
+
+        # ---- Print summary --------------------------------------------------------
+        if not quiet:
+            if summary.get("fell_back_to_full"):
+                console.print(
+                    "[yellow]Closure exceeded depth cap — fell back to full index.[/yellow]"
+                )
+            else:
+                console.print(
+                    f"[green]Resynced[/green] "
+                    f"+{summary['added']} added, "
+                    f"~{summary['modified']} modified, "
+                    f"-{summary['deleted']} deleted, "
+                    f"{summary['closure_resolved']} closure files re-resolved"
+                )
+
+
+def _get_head(root: Path) -> str:
+    """Return the current HEAD SHA for the git repo at root.
+
+    Raises typer.Exit(1) if git is unavailable or root is not a git repo.
+    """
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=str(root),
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            console.print(
+                f"[red]Could not determine HEAD SHA in {root}: {result.stderr.strip()}[/red]"
+            )
+            raise typer.Exit(1)
+        return result.stdout.strip()
+    except FileNotFoundError:
+        console.print("[red]git is not available — cannot determine HEAD SHA[/red]")
+        raise typer.Exit(1) from None