nao-core 0.0.38__py3-none-manylinux2014_aarch64.whl

nao_core/commands/sync/providers/databases/databricks.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_databricks(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Databricks database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_name = db_config.get_database_name()
+    db_path = base_path / "type=databricks" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/duckdb.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_duckdb(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync DuckDB database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+
+    db_name = db_config.get_database_name()
+    db_path = base_path / "type=duckdb" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    # List all schemas in DuckDB
+    schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/postgres.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_postgres(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync PostgreSQL database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_name = db_config.get_database_name()
+    db_path = base_path / "type=postgres" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema_name:
+        schemas = [db_config.schema_name]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/databases/provider.py

@@ -0,0 +1,129 @@
+"""Database sync provider implementation."""
+
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState, cleanup_stale_databases, cleanup_stale_paths
+from nao_core.commands.sync.registry import get_accessors
+from nao_core.config import AnyDatabaseConfig, NaoConfig
+
+from ..base import SyncProvider, SyncResult
+from .bigquery import sync_bigquery
+from .databricks import sync_databricks
+from .duckdb import sync_duckdb
+from .postgres import sync_postgres
+from .snowflake import sync_snowflake
+
+console = Console()
+
+# Registry mapping database types to their sync functions
+DATABASE_SYNC_FUNCTIONS = {
+    "bigquery": sync_bigquery,
+    "duckdb": sync_duckdb,
+    "databricks": sync_databricks,
+    "snowflake": sync_snowflake,
+    "postgres": sync_postgres,
+}
+
+
+class DatabaseSyncProvider(SyncProvider):
+    """Provider for syncing database schemas to markdown documentation."""
+
+    @property
+    def name(self) -> str:
+        return "Databases"
+
+    @property
+    def emoji(self) -> str:
+        return "🗄️"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "databases"
+
+    def pre_sync(self, config: NaoConfig, output_path: Path) -> None:
+        """
+        Always run before syncing.
+        """
+        cleanup_stale_databases(config.databases, output_path, verbose=True)
+
+    def get_items(self, config: NaoConfig) -> list[AnyDatabaseConfig]:
+        return config.databases
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured databases.
+
+        Args:
+                items: List of database configurations
+                output_path: Base path where database schemas are stored
+                project_path: Path to the nao project root (for template resolution)
+
+        Returns:
+                SyncResult with datasets and tables synced
+        """
+        if not items:
+            console.print("\n[dim]No databases configured[/dim]")
+            return SyncResult(provider_name=self.name, items_synced=0)
+
+        # Set project path for template resolution
+        DataAccessor.set_project_path(project_path)
+
+        total_datasets = 0
+        total_tables = 0
+        total_removed = 0
+        sync_states: list[DatabaseSyncState] = []
+
+        console.print(f"\n[bold cyan]{self.emoji}  Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
+
+        with Progress(
+            SpinnerColumn(style="dim"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=30, style="dim", complete_style="cyan", finished_style="green"),
+            TaskProgressColumn(),
+            console=console,
+            transient=False,
+        ) as progress:
+            for db in items:
+                # Get accessors from database config
+                db_accessors = get_accessors(db.accessors)
+                accessor_names = [a.filename.replace(".md", "") for a in db_accessors]
+
+                try:
+                    console.print(f"[dim]{db.name} accessors:[/dim] {', '.join(accessor_names)}")
+
+                    sync_fn = DATABASE_SYNC_FUNCTIONS.get(db.type)
+                    if sync_fn:
+                        state = sync_fn(db, output_path, progress, db_accessors)
+                        sync_states.append(state)
+                        total_datasets += state.schemas_synced
+                        total_tables += state.tables_synced
+                    else:
+                        console.print(f"[yellow]⚠ Unsupported database type: {db.type}[/yellow]")
+                except Exception as e:
+                    console.print(f"[bold red]✗[/bold red] Failed to sync {db.name}: {e}")
+
+        # Clean up stale files after all syncs complete
+        for state in sync_states:
+            removed = cleanup_stale_paths(state, verbose=True)
+            total_removed += removed
+
+        # Build summary
+        summary = f"{total_tables} tables across {total_datasets} datasets"
+        if total_removed > 0:
+            summary += f", {total_removed} stale removed"
+
+        return SyncResult(
+            provider_name=self.name,
+            items_synced=total_tables,
+            details={
+                "datasets": total_datasets,
+                "tables": total_tables,
+                "removed": total_removed,
+            },
+            summary=summary,
+        )

nao_core/commands/sync/providers/databases/snowflake.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_snowflake(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Snowflake database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_name = db_config.get_database_name()
+    db_path = base_path / "type=snowflake" / f"database={db_name}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/notion/__init__.py

@@ -0,0 +1,5 @@
+"""Notion syncing functionality for syncing Notion pages and databases."""
+
+from .provider import NotionSyncProvider
+
+__all__ = ["NotionSyncProvider"]

nao_core/commands/sync/providers/notion/provider.py

@@ -0,0 +1,205 @@
+import re
+from pathlib import Path
+from typing import Any, cast
+
+from notion2md.exporter.block import StringExporter
+from notion_client import Client
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+
+from nao_core.config.base import NaoConfig
+from nao_core.config.notion import NotionConfig
+
+from ..base import SyncProvider, SyncResult
+
+console = Console()
+
+# Notion page IDs are 32-character hex strings (UUID without dashes)
+NOTION_PAGE_ID_PATTERN = re.compile(r"[a-f0-9]{32}")
+
+
+def cleanup_stale_pages(synced_files: set[str], output_path: Path, verbose: bool = False) -> int:
+    """Remove markdown files that were not synced.
+
+    Args:
+        synced_files: Set of filenames that were synced in this run.
+        output_path: Path where synced markdown files are stored.
+        verbose: Whether to print cleanup messages.
+
+    Returns:
+        Number of stale files removed.
+    """
+    if not output_path.exists():
+        return 0
+
+    removed_count = 0
+    for file_path in output_path.iterdir():
+        if file_path.is_file() and file_path.suffix == ".md":
+            if file_path.name not in synced_files:
+                file_path.unlink()
+                removed_count += 1
+                if verbose:
+                    console.print(f"  [dim red]removing stale page:[/dim red] {file_path.name}")
+
+    return removed_count
+
+
+# Pattern to match markdown images: ![alt](url)
+IMAGE_PATTERN = re.compile(r"!\[[^\]]*\]\([^)]+\)\n?")
+
+
+def strip_images(markdown: str) -> str:
+    """Replace markdown image references with a placeholder."""
+    return IMAGE_PATTERN.sub("[image]\n", markdown)
+
+
+def extract_page_id(page_url: str) -> str:
+    """Extract Notion page ID from a URL.
+
+    Handles URLs like:
+    - https://www.notion.so/naolabs/Conversational-analytics-2bfc7a70bc0680978900d1e85ece83a0
+    - https://www.notion.so/2bfc7a70bc0680978900d1e85ece83a0
+    - 2bfc7a70bc0680978900d1e85ece83a0 (raw ID)
+    """
+    match = NOTION_PAGE_ID_PATTERN.search(page_url)
+    if match:
+        return match.group(0)
+    raise ValueError(f"Could not extract Notion page ID from: {page_url}")
+
+
+def get_page_title(client: Client, page_id: str) -> str:
+    """Get the title of a Notion page."""
+    page = cast(dict[str, Any], client.pages.retrieve(page_id=page_id))
+    properties = page.get("properties", {})
+
+    # Try common title property names
+    for prop_name in ["title", "Title", "Name", "name", "Page"]:
+        if prop_name in properties:
+            title_prop = properties[prop_name]
+            if title_prop.get("type") == "title":
+                title_array = title_prop.get("title", [])
+                if title_array:
+                    return "".join(t.get("plain_text", "") for t in title_array)
+
+    # Fallback to page ID if no title found
+    return page_id
+
+
+def get_page_as_markdown(page_url: str, api_key: str) -> tuple[str, str]:
+    """Fetch a Notion page and convert it to markdown.
+
+    Returns:
+        Tuple of (title, markdown_content)
+    """
+    page_id = extract_page_id(page_url)
+
+    # Get page title for the filename
+    client = Client(auth=api_key)
+    title = get_page_title(client, page_id)
+
+    # Export to markdown string using notion2md
+    md_exporter = StringExporter(block_id=page_id, token=api_key)
+    markdown = md_exporter.export()
+
+    # Strip images since we can't read them
+    markdown = strip_images(markdown)
+
+    content = f"""---
+title: {title}
+id: {page_id}
+---
+
+{markdown}
+"""
+
+    return title, content
+
+
+class NotionSyncProvider(SyncProvider):
+    """Provider for syncing Notion pages and databases."""
+
+    @property
+    def name(self) -> str:
+        return "Notion"
+
+    @property
+    def emoji(self) -> str:
+        return "📝"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "docs/notion"
+
+    def get_items(self, config: NaoConfig) -> list[NotionConfig]:
+        return [config.notion] if config.notion else []
+
+    def sync(self, items: list[NotionConfig], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync Notion pages to local filesystem as markdown files.
+
+        Args:
+            items: Notion configuration with pages to sync.
+            output_path: Path where synced markdown files should be written.
+            project_path: Path to the nao project root.
+
+        Returns:
+            SyncResult with statistics about what was synced.
+        """
+        if not items:
+            console.print("\n[dim]No Notion pages configured[/dim]")
+            return SyncResult(provider_name=self.name, items_synced=0, summary="No Notion configurations configured")
+
+        notion_config = items[0]
+        output_path.mkdir(parents=True, exist_ok=True)
+        pages_synced = 0
+        synced_pages: list[str] = []
+        synced_files: set[str] = set()
+
+        console.print(f"\n[bold cyan]{self.emoji}  Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
+
+        api_key = notion_config.api_key
+        total_pages = len(notion_config.pages)
+
+        with Progress(
+            SpinnerColumn(style="dim"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=30, style="dim", complete_style="cyan", finished_style="green"),
+            TaskProgressColumn(),
+            console=console,
+            transient=False,
+        ) as progress:
+            task = progress.add_task("Syncing pages", total=total_pages)
+
+            for page_url in notion_config.pages:
+                try:
+                    title, markdown = get_page_as_markdown(page_url, api_key)
+
+                    # Sanitize title for filename
+                    safe_title = re.sub(r"[^\w\s-]", "", title).strip().replace(" ", "-").lower()
+                    filename = f"{safe_title}.md"
+
+                    with open(output_path / filename, "w") as f:
+                        f.write(markdown)
+
+                    pages_synced += 1
+                    synced_pages.append(title)
+                    synced_files.add(filename)
+                    progress.update(task, advance=1, description=f"Synced: {title}")
+                except Exception as e:
+                    console.print(f"[bold red]✗[/bold red] Failed to sync page {page_url}: {e}")
+                    progress.update(task, advance=1)
+
+        # Clean up stale pages
+        removed_count = cleanup_stale_pages(synced_files, output_path, verbose=True)
+
+        # Build summary
+        summary = f"{pages_synced} pages synced as markdown"
+        if removed_count > 0:
+            summary += f", {removed_count} stale removed"
+
+        return SyncResult(
+            provider_name=self.name,
+            items_synced=pages_synced,
+            details={"pages": synced_pages, "removed": removed_count},
+            summary=summary,
+        )

nao_core/commands/sync/providers/repositories/__init__.py

@@ -0,0 +1,5 @@
+"""Repository syncing functionality for cloning and pulling git repositories."""
+
+from .provider import RepositorySyncProvider
+
+__all__ = ["RepositorySyncProvider"]