pdbminebuilder 0.2.0__py3-none-any.whl

pdbminebuilder/__init__.py

@@ -0,0 +1,3 @@
+"""pdb-mine-builder - Build a Mine-schema database from PDB data."""
+
+__version__ = "0.2.0"

pdbminebuilder/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m pdbminebuilder."""
+
+from pdbminebuilder.cli import app
+
+if __name__ == "__main__":
+    app()

pdbminebuilder/cli.py

@@ -0,0 +1,369 @@
+"""CLI interface using typer + rich."""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from rich.console import Console
+
+from pdbminebuilder import __version__
+from pdbminebuilder.config import load_config
+
+app = typer.Typer(
+    name="pmb",
+    help="pdb-mine-builder - Build a Mine-schema database from PDB data.",
+    rich_markup_mode="rich",
+)
+console = Console()
+
+
+def setup_logging(log_file: Path | None, verbose: bool = False) -> logging.Logger:
+    """Configure logging with optional file output.
+
+    Args:
+        log_file: Path to log file (None for no file logging)
+        verbose: If True, set DEBUG level; otherwise INFO
+
+    Returns:
+        Configured logger
+    """
+    logger = logging.getLogger("pmb")
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+
+    # Clear existing handlers
+    logger.handlers.clear()
+
+    # Console handler (only warnings and errors)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.WARNING)
+    console_handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    logger.addHandler(console_handler)
+
+    # File handler (if specified)
+    if log_file:
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
+        file_handler.setLevel(logging.DEBUG if verbose else logging.INFO)
+        file_handler.setFormatter(
+            logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
+        )
+        logger.addHandler(file_handler)
+        console.print(f"[dim]Logging to: {log_file}[/dim]")
+
+    return logger
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        console.print(f"pmb version {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        Optional[bool],
+        typer.Option("--version", "-v", callback=version_callback, is_eager=True),
+    ] = None,
+) -> None:
+    """pdb-mine-builder - Build a Mine-schema database from PDB data."""
+    pass
+
+
+@app.command()
+def sync(
+    targets: Annotated[
+        Optional[list[str]],
+        typer.Argument(
+            help="Sync targets: pdbj, pdbj-json, cc, cc-json, ccmodel, ccmodel-json, prd, prd-json, prd-family, vrpt, contacts, schemas"
+        ),
+    ] = None,
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+    dry_run: Annotated[
+        bool,
+        typer.Option(
+            "--dry-run", "-n", help="Show what would be synced without actually syncing"
+        ),
+    ] = False,
+) -> None:
+    """Synchronize data from PDBj via rsync."""
+    from pdbminebuilder.commands.sync import run_sync
+
+    settings = load_config(config)
+    run_sync(settings, targets or [], dry_run=dry_run)
+
+
+@app.command()
+def update(
+    pipelines: Annotated[
+        Optional[list[str]],
+        typer.Argument(
+            help="Pipelines: pdbj, cc, ccmodel, prd, prd_family, vrpt, contacts, emdb, ihm (format via config)"
+        ),
+    ] = None,
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+    limit: Annotated[
+        Optional[int],
+        typer.Option("--limit", "-l", help="Limit number of entries to process"),
+    ] = None,
+    workers: Annotated[
+        Optional[int],
+        typer.Option(
+            "--workers", "-w", help="Number of worker processes (overrides config)"
+        ),
+    ] = None,
+    log: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--log",
+            help="Log file path (default: logs/<pipeline>_YYYYMMDD_HHMMSS.log)",
+        ),
+    ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option("--verbose", "-v", help="Enable verbose (DEBUG) logging"),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force",
+            "-f",
+            help="Reprocess all entries ignoring cached mtimes (pdbj, vrpt, contacts only)",
+        ),
+    ] = False,
+) -> None:
+    """Run database update pipelines."""
+    from pdbminebuilder.commands.update import run_update
+
+    # Setup logging with pipeline name in filename
+    if log is None:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        if pipelines and len(pipelines) == 1:
+            # Single pipeline: use pipeline name
+            log_name = pipelines[0].replace("-", "_")
+        elif pipelines:
+            # Multiple pipelines: use "multi"
+            log_name = "multi"
+        else:
+            # All pipelines
+            log_name = "all"
+        log = Path(f"logs/{log_name}_{timestamp}.log")
+    logger = setup_logging(log, verbose)
+
+    settings = load_config(config)
+    if workers is not None:
+        settings.rdb.nworkers = workers
+
+    logger.info(f"Starting update: pipelines={pipelines or 'all'}, limit={limit}")
+    run_update(settings, pipelines or [], limit=limit, force=force)
+    logger.info("Update completed")
+
+
+@app.command()
+def load(
+    pipelines: Annotated[
+        Optional[list[str]],
+        typer.Argument(
+            help="Pipelines: pdbj, cc, ccmodel, prd, prd_family, vrpt, contacts"
+        ),
+    ] = None,
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+    limit: Annotated[
+        Optional[int],
+        typer.Option("--limit", "-l", help="Limit number of entries to process"),
+    ] = None,
+    workers: Annotated[
+        Optional[int],
+        typer.Option(
+            "--workers", "-w", help="Number of worker processes (overrides config)"
+        ),
+    ] = None,
+    log: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--log",
+            help="Log file path (default: logs/load_<pipeline>_YYYYMMDD_HHMMSS.log)",
+        ),
+    ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option("--verbose", "-v", help="Enable verbose (DEBUG) logging"),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Skip TRUNCATE confirmation prompt"),
+    ] = False,
+) -> None:
+    """Bulk load data using COPY protocol (TRUNCATE + COPY).
+
+    Significantly faster than 'update' for initial/full database loads.
+    WARNING: This will TRUNCATE all tables in the target schema before loading.
+
+    Examples:
+        pmb load pdbj --limit 1000 --force
+        pmb load cc ccmodel prd --force
+    """
+    from pdbminebuilder.commands.load import run_load
+
+    if log is None:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        if pipelines and len(pipelines) == 1:
+            log_name = f"load_{pipelines[0].replace('-', '_')}"
+        elif pipelines:
+            log_name = "load_multi"
+        else:
+            log_name = "load_all"
+        log = Path(f"logs/{log_name}_{timestamp}.log")
+    logger = setup_logging(log, verbose)
+
+    settings = load_config(config)
+    if workers is not None:
+        settings.rdb.nworkers = workers
+
+    logger.info(f"Starting load: pipelines={pipelines or []}, limit={limit}")
+    run_load(settings, pipelines or [], limit=limit, force=force)
+    logger.info("Load completed")
+
+
+@app.command(name="all")
+def run_all(
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+) -> None:
+    """Run full sync and update cycle."""
+    from pdbminebuilder.commands.sync import run_sync
+    from pdbminebuilder.commands.update import run_update
+
+    settings = load_config(config)
+    console.print("[bold blue]Starting full sync and update cycle...[/bold blue]")
+
+    console.print("\n[bold]Phase 1: Sync[/bold]")
+    run_sync(settings, [], dry_run=False)
+
+    console.print("\n[bold]Phase 2: Update[/bold]")
+    run_update(settings, [])
+
+    console.print("\n[bold green]Full cycle completed![/bold green]")
+
+
+@app.command(name="setup-rdkit")
+def setup_rdkit(
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+) -> None:
+    """Setup RDKit extension and SQL functions.
+
+    Creates RDKit extension, mol column on cc.brief_summary,
+    and loads chemical search functions (similar_compounds, substructure_search, etc.).
+
+    This is automatically run by the cc pipeline, but can be run
+    independently to add functions to an existing database.
+    """
+    from pdbminebuilder.pipelines.cc import _ensure_rdkit_setup
+
+    settings = load_config(config)
+    console.print("[bold]Setting up RDKit extension and functions...[/bold]")
+    _ensure_rdkit_setup(settings.rdb.constring)
+    console.print("[bold green]RDKit setup completed![/bold green]")
+
+
+@app.command()
+def test(
+    pipelines: Annotated[
+        Optional[list[str]],
+        typer.Argument(help="Pipelines to test"),
+    ] = None,
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.test.yml"),
+    drop: Annotated[
+        bool,
+        typer.Option("--drop", "-d", help="Drop existing test database"),
+    ] = False,
+    limit: Annotated[
+        int,
+        typer.Option("--limit", "-l", help="Limit number of files to process"),
+    ] = 10,
+    workers: Annotated[
+        Optional[int],
+        typer.Option(
+            "--workers", "-w", help="Number of worker processes (overrides config)"
+        ),
+    ] = None,
+) -> None:
+    """Create test database and validate pipelines."""
+    from pdbminebuilder.commands.test import run_test
+
+    settings = load_config(config)
+    if workers is not None:
+        settings.rdb.nworkers = workers
+    run_test(settings, pipelines or [], drop=drop, limit=limit)
+
+
+@app.command()
+def reset(
+    schemas: Annotated[
+        Optional[list[str]],
+        typer.Argument(
+            help="Schemas to reset: pdbj, cc, ccmodel, prd, prd_family, vrpt, contacts, emdb, ihm (or 'all')"
+        ),
+    ] = None,
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Skip confirmation prompt"),
+    ] = False,
+) -> None:
+    """Drop and reset database schemas (for testing/reloading).
+
+    Examples:
+        pmb reset cc          # Reset cc schema only
+        pmb reset cc pdbj     # Reset cc and pdbj schemas
+        pmb reset all         # Reset ALL schemas (dangerous!)
+        pmb reset all -f      # Reset all without confirmation
+    """
+    from pdbminebuilder.commands.reset import run_reset
+
+    settings = load_config(config)
+    run_reset(settings, schemas or [], force=force)
+
+
+@app.command()
+def stats(
+    config: Annotated[
+        Path,
+        typer.Option("--config", "-c", help="Config file path"),
+    ] = Path("config.yml"),
+) -> None:
+    """Show database statistics.
+
+    Displays table counts, row counts, and last update timestamps
+    for each schema in the database.
+    """
+    from pdbminebuilder.commands.stats import run_stats
+
+    settings = load_config(config)
+    run_stats(settings)
+
+
+if __name__ == "__main__":
+    app()

pdbminebuilder/commands/__init__.py

@@ -0,0 +1 @@
+"""CLI commands."""

pdbminebuilder/commands/load.py

@@ -0,0 +1,164 @@
+"""Load command - bulk load data using COPY protocol."""
+
+import importlib
+from typing import Any, Callable
+
+import typer
+from rich.console import Console
+
+from pdbminebuilder.commands.update import DUAL_FORMAT_PIPELINES, LEGACY_ALIASES
+from pdbminebuilder.commands.utils import resolve_legacy_aliases
+from pdbminebuilder.config import Settings
+from pdbminebuilder.db.connection import close_pool, init_pool
+from pdbminebuilder.db.loader import LoaderResult, ensure_schema, truncate_schema_tables
+from pdbminebuilder.db.metadata import (
+    ensure_entry_metadata_table,
+    ensure_metadata_table,
+    update_pipeline_metadata,
+)
+from pdbminebuilder.models import get_metadata
+
+console = Console()
+
+# Pipelines supported by load command.
+# Each pipeline module must expose run_cif_load() for CIF format.
+# Dual-format pipelines (DUAL_FORMAT_PIPELINES) must also expose
+# run_load() for mmJSON format.
+LOAD_PIPELINES = ["pdbj", "cc", "ccmodel", "prd", "prd_family", "vrpt", "contacts"]
+
+
+def _get_load_runner(
+    pipeline_name: str, settings: Settings
+) -> Callable[..., list[LoaderResult]]:
+    """Get the load runner function for a pipeline.
+
+    For dual-format pipelines, reads format from config:
+      - format=cif    -> run_cif_load()
+      - format=mmjson -> run_load()
+
+    Other pipelines always use run_cif_load().
+
+    Returns:
+        Callable with signature (settings, config, meta, limit=...) -> list[LoaderResult]
+
+    Raises:
+        RuntimeError: If the pipeline module cannot be imported or the
+            required load function is missing.
+    """
+    try:
+        pipeline_module = importlib.import_module(
+            f"pdbminebuilder.pipelines.{pipeline_name}"
+        )
+    except ImportError as e:
+        raise RuntimeError(
+            f"Failed to import pipeline module 'pdbminebuilder.pipelines.{pipeline_name}': {e}. "
+            f"Check that all required dependencies are installed."
+        ) from e
+
+    if pipeline_name in DUAL_FORMAT_PIPELINES:
+        pipeline_config = settings.pipelines.get(pipeline_name)
+        if pipeline_config and pipeline_config.format == "mmjson":
+            runner = getattr(pipeline_module, "run_load", None)
+            if runner is None:
+                raise RuntimeError(
+                    f"Pipeline '{pipeline_name}' does not support mmJSON load mode "
+                    f"(missing run_load in pdbminebuilder.pipelines.{pipeline_name}). "
+                    f"Set format='cif' in config.yml or implement run_load()."
+                )
+            return runner
+
+    runner = getattr(pipeline_module, "run_cif_load", None)
+    if runner is None:
+        raise RuntimeError(
+            f"Pipeline '{pipeline_name}' does not support load mode "
+            f"(missing run_cif_load in pdbminebuilder.pipelines.{pipeline_name})."
+        )
+    return runner
+
+
+def run_load(
+    settings: Settings,
+    pipelines: list[str],
+    limit: int | None = None,
+    force: bool = False,
+) -> None:
+    """Run bulk load pipelines (TRUNCATE + COPY).
+
+    Args:
+        settings: Application settings
+        pipelines: List of pipeline names to run
+        limit: Optional limit on number of entries to process
+        force: Skip interactive TRUNCATE confirmation
+    """
+    if not pipelines:
+        console.print("[red]No pipelines specified.[/red]")
+        console.print(f"[dim]Available: {', '.join(LOAD_PIPELINES)}[/dim]")
+        return
+
+    # Resolve legacy aliases with deprecation warnings
+    pipelines = resolve_legacy_aliases(pipelines, LEGACY_ALIASES, "Pipeline")
+
+    invalid = [p for p in pipelines if p not in LOAD_PIPELINES]
+    if invalid:
+        console.print(f"[red]Invalid pipelines: {', '.join(invalid)}[/red]")
+        console.print(f"[dim]Available: {', '.join(LOAD_PIPELINES)}[/dim]")
+        return
+
+    # Confirmation prompt unless --force
+    if not force:
+        schema_names = sorted(set(pipelines))
+        console.print(
+            f"[bold red]WARNING: This will TRUNCATE all tables in: "
+            f"{', '.join(schema_names)}[/bold red]"
+        )
+        typer.confirm("Continue?", abort=True)
+
+    console.print(f"[bold]Loading {len(pipelines)} pipeline(s)...[/bold]")
+
+    # Pre-flight: verify all pipelines are importable and configured
+    # BEFORE truncating any data.
+    pipeline_runners: list[tuple[str, Any, Any, Any]] = []
+
+    for pipeline_name in pipelines:
+        pipeline_config = settings.pipelines.get(pipeline_name)
+        if not pipeline_config:
+            msg = (
+                f"Pipeline {pipeline_name!r} has no configuration in "
+                f"settings.pipelines. Check config.yml."
+            )
+            raise RuntimeError(msg)
+
+        runner = _get_load_runner(pipeline_name, settings)
+        meta = get_metadata(pipeline_name)
+        pipeline_runners.append((pipeline_name, pipeline_config, meta, runner))
+
+    init_pool(settings.rdb.constring, max_size=settings.rdb.get_workers() + 2)
+    ensure_metadata_table(settings.rdb.constring)
+    ensure_entry_metadata_table(settings.rdb.constring)
+
+    try:
+        for pipeline_name, pipeline_config, meta, runner in pipeline_runners:
+            console.print(f"\n[bold blue]Pipeline: {pipeline_name} (load)[/bold blue]")
+            console.print(f"  Schema: {meta.schema}")
+            console.print(f"  Tables: {len(meta.tables)}")
+
+            # Ensure schema exists
+            ensure_schema(meta, settings.rdb.constring)
+
+            # TRUNCATE all tables
+            truncate_schema_tables(meta, settings.rdb.constring)
+
+            # Run load pipeline
+            results = runner(settings, pipeline_config, meta, limit=limit)
+
+            success_count = sum(1 for r in results if r.success) if results else None
+            update_pipeline_metadata(
+                settings.rdb.constring,
+                meta.schema,
+                entries_count=success_count,
+            )
+
+    finally:
+        close_pool()
+
+    console.print("\n[bold green]Load completed![/bold green]")

pdbminebuilder/commands/reset.py

@@ -0,0 +1,101 @@
+"""Reset command - drop and reset database schemas."""
+
+import psycopg
+from rich.console import Console
+from rich.prompt import Confirm
+
+from pdbminebuilder.config import Settings
+
+console = Console()
+
+# Known schemas that can be reset
+KNOWN_SCHEMAS = [
+    "pdbj",
+    "cc",
+    "ccmodel",
+    "prd",
+    "prd_family",
+    "vrpt",
+    "contacts",
+    "emdb",
+    "ihm",
+]
+
+
+def run_reset(
+    settings: Settings,
+    schemas: list[str],
+    force: bool = False,
+) -> None:
+    """Drop specified schemas from the database.
+
+    Args:
+        settings: Application settings
+        schemas: List of schema names to drop, or ["all"] for all schemas
+        force: Skip confirmation prompt
+    """
+    if not schemas:
+        console.print("[yellow]No schemas specified. Available schemas:[/yellow]")
+        console.print(f"  {', '.join(KNOWN_SCHEMAS)}")
+        console.print("\nUsage:")
+        console.print("  pmb reset cc          # Reset cc schema")
+        console.print("  pmb reset cc pdbj     # Reset multiple schemas")
+        console.print("  pmb reset all         # Reset ALL schemas")
+        return
+
+    # Handle 'all' keyword
+    if "all" in schemas:
+        target_schemas = KNOWN_SCHEMAS.copy()
+    else:
+        # Validate schema names
+        invalid = [s for s in schemas if s not in KNOWN_SCHEMAS]
+        if invalid:
+            console.print(f"[red]Unknown schema(s): {', '.join(invalid)}[/red]")
+            console.print(f"[yellow]Valid schemas: {', '.join(KNOWN_SCHEMAS)}[/yellow]")
+            return
+        target_schemas = schemas
+
+    # Show what will be dropped
+    console.print(
+        "\n[bold red]WARNING: This will DROP the following schemas:[/bold red]"
+    )
+    for schema in target_schemas:
+        console.print(f"  • {schema}")
+    console.print(
+        "\n[yellow]All data in these schemas will be permanently deleted![/yellow]"
+    )
+
+    # Confirm unless --force
+    if not force:
+        confirmed = Confirm.ask("\nAre you sure you want to continue?", default=False)
+        if not confirmed:
+            console.print("[dim]Aborted.[/dim]")
+            return
+
+    # Drop schemas
+    with psycopg.connect(settings.rdb.constring) as conn:
+        with conn.cursor() as cur:
+            for schema in target_schemas:
+                try:
+                    # Check if schema exists
+                    cur.execute(
+                        "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = %s)",
+                        (schema,),
+                    )
+                    result = cur.fetchone()
+                    exists = result[0] if result else False
+
+                    if exists:
+                        cur.execute(
+                            f"DROP SCHEMA {schema} CASCADE"  # type: ignore[arg-type]
+                        )
+                        console.print(f"  [green]✓[/green] Dropped schema: {schema}")
+                    else:
+                        console.print(f"  [dim]○[/dim] Schema not found: {schema}")
+                except Exception as e:
+                    console.print(f"  [red]✗[/red] Error dropping {schema}: {e}")
+
+        conn.commit()
+
+    console.print("\n[bold green]Reset completed![/bold green]")
+    console.print("[dim]Run 'pmb update <pipeline>' to reload data.[/dim]")