nao-core 0.0.30__py3-none-any.whl → 0.0.31__py3-none-any.whl

nao_core/commands/sync/cleanup.py

@@ -0,0 +1,133 @@
+"""Cleanup utilities for removing stale sync files."""
+
+import shutil
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from rich.console import Console
+
+console = Console()
+
+
+@dataclass
+class DatabaseSyncState:
+    """Tracks the state of a database sync operation.
+
+    Used to track which paths were synced so stale paths can be cleaned up.
+    """
+
+    db_path: Path
+    """The root path for this database (e.g., databases/type=duckdb/database=mydb)"""
+
+    synced_schemas: set[str] = field(default_factory=set)
+    """Set of schema names that were synced"""
+
+    synced_tables: dict[str, set[str]] = field(default_factory=dict)
+    """Dict mapping schema names to sets of table names that were synced"""
+
+    schemas_synced: int = 0
+    """Count of schemas synced"""
+
+    tables_synced: int = 0
+    """Count of tables synced"""
+
+    def add_table(self, schema: str, table: str) -> None:
+        """Record that a table was synced.
+
+        Args:
+            schema: The schema/dataset name
+            table: The table name
+        """
+        self.synced_schemas.add(schema)
+        if schema not in self.synced_tables:
+            self.synced_tables[schema] = set()
+        self.synced_tables[schema].add(table)
+        self.tables_synced += 1
+
+    def add_schema(self, schema: str) -> None:
+        """Record that a schema was synced (even if empty).
+
+        Args:
+            schema: The schema/dataset name
+        """
+        self.synced_schemas.add(schema)
+        self.schemas_synced += 1
+
+
+def cleanup_stale_paths(state: DatabaseSyncState, verbose: bool = False) -> int:
+    """Remove directories that exist on disk but weren't synced.
+
+    This function cleans up:
+    - Table directories that no longer exist in the source
+    - Schema directories that no longer exist or have no tables
+
+    Args:
+        state: The sync state tracking what was synced
+        verbose: Whether to print cleanup messages
+
+    Returns:
+        Number of stale paths removed
+    """
+    removed_count = 0
+
+    if not state.db_path.exists():
+        return 0
+
+    # Find all existing schema directories
+    existing_schemas = {
+        d.name.replace("schema=", ""): d for d in state.db_path.iterdir() if d.is_dir() and d.name.startswith("schema=")
+    }
+
+    # Remove schemas that weren't synced
+    for schema_name, schema_path in existing_schemas.items():
+        if schema_name not in state.synced_schemas:
+            if verbose:
+                console.print(f"  [dim red]removing stale schema:[/dim red] {schema_name}")
+            shutil.rmtree(schema_path)
+            removed_count += 1
+            continue
+
+        # Find existing tables in this schema
+        existing_tables = {
+            d.name.replace("table=", ""): d for d in schema_path.iterdir() if d.is_dir() and d.name.startswith("table=")
+        }
+
+        synced_tables_for_schema = state.synced_tables.get(schema_name, set())
+
+        # Remove tables that weren't synced
+        for table_name, table_path in existing_tables.items():
+            if table_name not in synced_tables_for_schema:
+                if verbose:
+                    console.print(f"  [dim red]removing stale table:[/dim red] {schema_name}.{table_name}")
+                shutil.rmtree(table_path)
+                removed_count += 1
+
+    return removed_count
+
+
+def cleanup_stale_database_types(base_path: Path, active_db_types: set[str], verbose: bool = False) -> int:
+    """Remove database type directories that are no longer configured.
+
+    Args:
+        base_path: The base databases output path
+        active_db_types: Set of database type directory names that should exist
+                         (e.g., {'type=duckdb', 'type=postgres'})
+        verbose: Whether to print cleanup messages
+
+    Returns:
+        Number of stale database type directories removed
+    """
+    removed_count = 0
+
+    if not base_path.exists():
+        return 0
+
+    for db_type_dir in base_path.iterdir():
+        if db_type_dir.is_dir() and db_type_dir.name.startswith("type="):
+            if db_type_dir.name not in active_db_types:
+                if verbose:
+                    console.print(f"  [dim red]removing stale database type:[/dim red] {db_type_dir.name}")
+                shutil.rmtree(db_type_dir)
+                removed_count += 1
+
+    return removed_count

nao_core/commands/sync/providers/__init__.py

@@ -0,0 +1,30 @@
+"""Sync providers for different resource types."""
+
+from .base import SyncProvider, SyncResult
+from .databases.provider import DatabaseSyncProvider
+from .repositories.provider import RepositorySyncProvider
+
+# Default providers in order of execution
+DEFAULT_PROVIDERS: list[SyncProvider] = [
+    RepositorySyncProvider(),
+    DatabaseSyncProvider(),
+]
+
+
+def get_all_providers() -> list[SyncProvider]:
+    """Get all registered sync providers.
+
+    Returns:
+            List of sync provider instances
+    """
+    return DEFAULT_PROVIDERS.copy()
+
+
+__all__ = [
+    "SyncProvider",
+    "SyncResult",
+    "DatabaseSyncProvider",
+    "RepositorySyncProvider",
+    "DEFAULT_PROVIDERS",
+    "get_all_providers",
+]

nao_core/commands/sync/providers/base.py

@@ -0,0 +1,87 @@
+"""Base class for sync providers."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from nao_core.config import NaoConfig
+
+
+@dataclass
+class SyncResult:
+    """Result of a sync operation."""
+
+    provider_name: str
+    items_synced: int
+    details: dict[str, Any] | None = None
+    summary: str | None = None
+
+    def get_summary(self) -> str:
+        """Get a human-readable summary of the sync result."""
+        if self.summary:
+            return self.summary
+        return f"{self.items_synced} synced"
+
+
+class SyncProvider(ABC):
+    """Abstract base class for sync providers.
+
+    A sync provider is responsible for synchronizing a specific type of resource
+    (e.g., repositories, databases) from the nao configuration to local files.
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable name for this provider (e.g., 'Repositories', 'Databases')."""
+        ...
+
+    @property
+    @abstractmethod
+    def emoji(self) -> str:
+        """Emoji icon for this provider."""
+        ...
+
+    @property
+    @abstractmethod
+    def default_output_dir(self) -> str:
+        """Default output directory for this provider."""
+        ...
+
+    @abstractmethod
+    def get_items(self, config: NaoConfig) -> list[Any]:
+        """Extract items to sync from the configuration.
+
+        Args:
+                config: The nao configuration
+
+        Returns:
+                List of items to sync (e.g., repo configs, database configs)
+        """
+        ...
+
+    @abstractmethod
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync the items to the output path.
+
+        Args:
+                items: List of items to sync
+                output_path: Path where synced data should be written
+                project_path: Path to the nao project root (for template resolution)
+
+        Returns:
+                SyncResult with statistics about what was synced
+        """
+        ...
+
+    def should_sync(self, config: NaoConfig) -> bool:
+        """Check if this provider has items to sync.
+
+        Args:
+                config: The nao configuration
+
+        Returns:
+                True if there are items to sync
+        """
+        return len(self.get_items(config)) > 0

nao_core/commands/sync/providers/databases/__init__.py

@@ -0,0 +1,17 @@
+"""Database syncing functionality for generating markdown documentation from database schemas."""
+
+from .bigquery import sync_bigquery
+from .databricks import sync_databricks
+from .duckdb import sync_duckdb
+from .postgres import sync_postgres
+from .provider import DatabaseSyncProvider
+from .snowflake import sync_snowflake
+
+__all__ = [
+    "DatabaseSyncProvider",
+    "sync_bigquery",
+    "sync_databricks",
+    "sync_duckdb",
+    "sync_postgres",
+    "sync_snowflake",
+]

nao_core/commands/sync/providers/databases/bigquery.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_bigquery(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync BigQuery database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=bigquery" / f"database={db_config.project_id}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.dataset_id:
+        datasets = [db_config.dataset_id]
+    else:
+        datasets = conn.list_databases()
+
+    dataset_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(datasets),
+    )
+
+    for dataset in datasets:
+        try:
+            all_tables = conn.list_tables(database=dataset)
+        except Exception:
+            progress.update(dataset_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(dataset, t)]
+
+        # Skip dataset if no tables match
+        if not tables:
+            progress.update(dataset_task, advance=1)
+            continue
+
+        dataset_path = db_path / f"schema={dataset}"
+        dataset_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(dataset)
+
+        table_task = progress.add_task(
+            f"  [cyan]{dataset}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = dataset_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, dataset, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(dataset, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(dataset_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/databricks.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_databricks(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Databricks database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    catalog = db_config.catalog or "main"
+    db_path = base_path / "type=databricks" / f"database={catalog}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/duckdb.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_duckdb(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync DuckDB database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+
+    # Derive database name from path
+    if db_config.path == ":memory:":
+        db_name = "memory"
+    else:
+        db_name = Path(db_config.path).stem
+
+    db_path = base_path / "type=duckdb" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    # List all schemas in DuckDB
+    schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/postgres.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_postgres(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync PostgreSQL database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=postgres" / f"database={db_config.database}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema_name:
+        schemas = [db_config.schema_name]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state