nao-core 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

nao_core/commands/sync/providers/databases/__init__.py

@@ -0,0 +1,17 @@
+"""Database syncing functionality for generating markdown documentation from database schemas."""
+
+from .bigquery import sync_bigquery
+from .databricks import sync_databricks
+from .duckdb import sync_duckdb
+from .postgres import sync_postgres
+from .provider import DatabaseSyncProvider
+from .snowflake import sync_snowflake
+
+__all__ = [
+    "DatabaseSyncProvider",
+    "sync_bigquery",
+    "sync_databricks",
+    "sync_duckdb",
+    "sync_postgres",
+    "sync_snowflake",
+]

nao_core/commands/sync/providers/databases/bigquery.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_bigquery(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync BigQuery database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=bigquery" / f"database={db_config.project_id}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.dataset_id:
+        datasets = [db_config.dataset_id]
+    else:
+        datasets = conn.list_databases()
+
+    dataset_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(datasets),
+    )
+
+    for dataset in datasets:
+        try:
+            all_tables = conn.list_tables(database=dataset)
+        except Exception:
+            progress.update(dataset_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(dataset, t)]
+
+        # Skip dataset if no tables match
+        if not tables:
+            progress.update(dataset_task, advance=1)
+            continue
+
+        dataset_path = db_path / f"schema={dataset}"
+        dataset_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(dataset)
+
+        table_task = progress.add_task(
+            f"  [cyan]{dataset}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = dataset_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, dataset, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(dataset, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(dataset_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/databricks.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_databricks(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Databricks database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    catalog = db_config.catalog or "main"
+    db_path = base_path / "type=databricks" / f"database={catalog}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/duckdb.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_duckdb(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync DuckDB database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+
+    # Derive database name from path
+    if db_config.path == ":memory:":
+        db_name = "memory"
+    else:
+        db_name = Path(db_config.path).stem
+
+    db_path = base_path / "type=duckdb" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    # List all schemas in DuckDB
+    schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/postgres.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_postgres(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync PostgreSQL database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=postgres" / f"database={db_config.database}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema_name:
+        schemas = [db_config.schema_name]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/provider.py

@@ -0,0 +1,123 @@
+"""Database sync provider implementation."""
+
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState, cleanup_stale_paths
+from nao_core.commands.sync.registry import get_accessors
+from nao_core.config import AnyDatabaseConfig, NaoConfig
+
+from ..base import SyncProvider, SyncResult
+from .bigquery import sync_bigquery
+from .databricks import sync_databricks
+from .duckdb import sync_duckdb
+from .postgres import sync_postgres
+from .snowflake import sync_snowflake
+
+console = Console()
+
+# Registry mapping database types to their sync functions
+DATABASE_SYNC_FUNCTIONS = {
+    "bigquery": sync_bigquery,
+    "duckdb": sync_duckdb,
+    "databricks": sync_databricks,
+    "snowflake": sync_snowflake,
+    "postgres": sync_postgres,
+}
+
+
+class DatabaseSyncProvider(SyncProvider):
+    """Provider for syncing database schemas to markdown documentation."""
+
+    @property
+    def name(self) -> str:
+        return "Databases"
+
+    @property
+    def emoji(self) -> str:
+        return "🗄️"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "databases"
+
+    def get_items(self, config: NaoConfig) -> list[AnyDatabaseConfig]:
+        return config.databases
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured databases.
+
+        Args:
+                items: List of database configurations
+                output_path: Base path where database schemas are stored
+                project_path: Path to the nao project root (for template resolution)
+
+        Returns:
+                SyncResult with datasets and tables synced
+        """
+        if not items:
+            console.print("\n[dim]No databases configured[/dim]")
+            return SyncResult(provider_name=self.name, items_synced=0)
+
+        # Set project path for template resolution
+        DataAccessor.set_project_path(project_path)
+
+        total_datasets = 0
+        total_tables = 0
+        total_removed = 0
+        sync_states: list[DatabaseSyncState] = []
+
+        console.print(f"\n[bold cyan]{self.emoji}  Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
+
+        with Progress(
+            SpinnerColumn(style="dim"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=30, style="dim", complete_style="cyan", finished_style="green"),
+            TaskProgressColumn(),
+            console=console,
+            transient=False,
+        ) as progress:
+            for db in items:
+                # Get accessors from database config
+                db_accessors = get_accessors(db.accessors)
+                accessor_names = [a.filename.replace(".md", "") for a in db_accessors]
+
+                try:
+                    console.print(f"[dim]{db.name} accessors:[/dim] {', '.join(accessor_names)}")
+
+                    sync_fn = DATABASE_SYNC_FUNCTIONS.get(db.type)
+                    if sync_fn:
+                        state = sync_fn(db, output_path, progress, db_accessors)
+                        sync_states.append(state)
+                        total_datasets += state.schemas_synced
+                        total_tables += state.tables_synced
+                    else:
+                        console.print(f"[yellow]⚠ Unsupported database type: {db.type}[/yellow]")
+                except Exception as e:
+                    console.print(f"[bold red]✗[/bold red] Failed to sync {db.name}: {e}")
+
+        # Clean up stale files after all syncs complete
+        for state in sync_states:
+            removed = cleanup_stale_paths(state, verbose=True)
+            total_removed += removed
+
+        # Build summary
+        summary = f"{total_tables} tables across {total_datasets} datasets"
+        if total_removed > 0:
+            summary += f", {total_removed} stale removed"
+
+        return SyncResult(
+            provider_name=self.name,
+            items_synced=total_tables,
+            details={
+                "datasets": total_datasets,
+                "tables": total_tables,
+                "removed": total_removed,
+            },
+            summary=summary,
+        )

nao_core/commands/sync/providers/databases/snowflake.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_snowflake(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Snowflake database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=snowflake" / f"database={db_config.database}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/repositories/__init__.py

@@ -0,0 +1,5 @@
+"""Repository syncing functionality for cloning and pulling git repositories."""
+
+from .provider import RepositorySyncProvider
+
+__all__ = ["RepositorySyncProvider"]

nao_core/commands/sync/{repositories.py → providers/repositories/provider.py}

@@ -1,12 +1,16 @@
-"""Repository syncing functionality for cloning and pulling git repositories."""
+"""Repository sync provider implementation."""
 
 import subprocess
 from pathlib import Path
+from typing import Any
 
 from rich.console import Console
 
+from nao_core.config import NaoConfig
 from nao_core.config.repos import RepoConfig
 
+from ..base import SyncProvider, SyncResult
+
 console = Console()
 
 
@@ -76,28 +80,47 @@ def clone_or_pull_repo(repo: RepoConfig, base_path: Path) -> bool:
         return False
 
 
-def sync_repositories(repos: list[RepoConfig], base_path: Path) -> int:
-    """Sync all configured repositories.
+class RepositorySyncProvider(SyncProvider):
+    """Provider for syncing git repositories."""
 
-    Args:
-            repos: List of repository configurations
-            base_path: Base path where repositories are stored
+    @property
+    def name(self) -> str:
+        return "Repositories"
 
-    Returns:
-            Number of successfully synced repositories
-    """
-    if not repos:
-        return 0
+    @property
+    def emoji(self) -> str:
+        return "📦"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "repos"
+
+    def get_items(self, config: NaoConfig) -> list[RepoConfig]:
+        return config.repos
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured repositories.
+
+        Args:
+                items: List of repository configurations
+                output_path: Base path where repositories are stored
+                project_path: Path to the nao project root (unused for repos)
+
+        Returns:
+                SyncResult with number of successfully synced repositories
+        """
+        if not items:
+            return SyncResult(provider_name=self.name, items_synced=0)
 
-    base_path.mkdir(parents=True, exist_ok=True)
-    success_count = 0
+        output_path.mkdir(parents=True, exist_ok=True)
+        success_count = 0
 
-    console.print("\n[bold cyan]📦 Syncing Repositories[/bold cyan]")
-    console.print(f"[dim]Location:[/dim] {base_path.absolute()}\n")
+        console.print(f"\n[bold cyan]{self.emoji} Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
 
-    for repo in repos:
-        if clone_or_pull_repo(repo, base_path):
-            success_count += 1
-            console.print(f"  [green]✓[/green] {repo.name}")
+        for repo in items:
+            if clone_or_pull_repo(repo, output_path):
+                success_count += 1
+                console.print(f"  [green]✓[/green] {repo.name}")
 
-    return success_count
+        return SyncResult(provider_name=self.name, items_synced=success_count)

nao_core/config/__init__.py

@@ -1,5 +1,15 @@
 from .base import NaoConfig
-from .databases import AccessorType, AnyDatabaseConfig, BigQueryConfig, DatabaseType
+from .databases import (
+    AccessorType,
+    AnyDatabaseConfig,
+    BigQueryConfig,
+    DatabaseType,
+    DatabricksConfig,
+    DuckDBConfig,
+    PostgresConfig,
+    SnowflakeConfig,
+)
+from .exceptions import InitError
 from .llm import LLMConfig, LLMProvider
 from .slack import SlackConfig
 
@@ -8,8 +18,13 @@ __all__ = [
     "AccessorType",
     "AnyDatabaseConfig",
     "BigQueryConfig",
+    "DuckDBConfig",
+    "DatabricksConfig",
+    "SnowflakeConfig",
+    "PostgresConfig",
     "DatabaseType",
     "LLMConfig",
     "LLMProvider",
     "SlackConfig",
+    "InitError",
 ]