nao-core 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

nao_core/commands/sync/accessors.py

@@ -1,13 +1,28 @@
 """Data accessor classes for generating markdown documentation from database tables."""
 
-import json
 from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
 
 from ibis import BaseBackend
 
+from nao_core.templates import get_template_engine
+
 
 class DataAccessor(ABC):
-    """Base class for data accessors that generate markdown files for tables."""
+    """Base class for data accessors that generate markdown files for tables.
+
+    Accessors use Jinja2 templates for generating output. Default templates
+    are shipped with nao and can be overridden by users by placing templates
+    with the same name in their project's `templates/` directory.
+
+    Example:
+        To override the preview template, create:
+        `<project_root>/templates/databases/preview.md.j2`
+    """
+
+    # Path to the nao project root (set by sync provider)
+    _project_path: Path | None = None
 
     @property
     @abstractmethod
@@ -15,24 +30,56 @@ class DataAccessor(ABC):
         """The filename this accessor writes to (e.g., 'columns.md')."""
         ...
 
+    @property
     @abstractmethod
-    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
-        """Generate the markdown content for a table.
+    def template_name(self) -> str:
+        """The template file to use (e.g., 'databases/columns.md.j2')."""
+        ...
+
+    @abstractmethod
+    def get_context(self, conn: BaseBackend, dataset: str, table: str) -> dict[str, Any]:
+        """Get the template context for rendering.
 
         Args:
-                conn: The Ibis database connection
-                dataset: The dataset/schema name
-                table: The table name
+            conn: The Ibis database connection
+            dataset: The dataset/schema name
+            table: The table name
 
         Returns:
-                Markdown string content
+            Dictionary of variables to pass to the template
         """
         ...
 
+    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
+        """Generate the markdown content for a table using templates.
+
+        Args:
+            conn: The Ibis database connection
+            dataset: The dataset/schema name
+            table: The table name
+
+        Returns:
+            Markdown string content
+        """
+        try:
+            context = self.get_context(conn, dataset, table)
+            engine = get_template_engine(self._project_path)
+            return engine.render(self.template_name, **context)
+        except Exception as e:
+            return f"# {table}\n\nError generating content: {e}"
+
     def get_table(self, conn: BaseBackend, dataset: str, table: str):
         """Helper to get an Ibis table reference."""
-        full_table_name = f"{dataset}.{table}"
-        return conn.table(full_table_name)
+        return conn.table(table, database=dataset)
+
+    @classmethod
+    def set_project_path(cls, path: Path | None) -> None:
+        """Set the project path for template resolution.
+
+        Args:
+            path: Path to the nao project root
+        """
+        cls._project_path = path
 
 
 def truncate_middle(text: str, max_length: int) -> str:
@@ -44,7 +91,14 @@ def truncate_middle(text: str, max_length: int) -> str:
 
 
 class ColumnsAccessor(DataAccessor):
-    """Generates columns.md with column names, types, and nullable info."""
+    """Generates columns.md with column names, types, and nullable info.
+
+    Template variables:
+        - table_name: Name of the table
+        - dataset: Schema/dataset name
+        - columns: List of dicts with 'name', 'type', 'nullable', 'description'
+        - column_count: Total number of columns
+    """
 
     def __init__(self, max_description_length: int = 256):
         self.max_description_length = max_description_length
@@ -53,37 +107,43 @@ class ColumnsAccessor(DataAccessor):
     def filename(self) -> str:
         return "columns.md"
 
-    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
-        try:
-            t = self.get_table(conn, dataset, table)
-            schema = t.schema()
-
-            columns = list(schema.items())
-
-            lines = [
-                f"# {table}",
-                "",
-                f"**Dataset:** `{dataset}`",
-                "",
-                f"## Columns ({len(columns)})",
-                "",
-            ]
-
-            for name, dtype in columns:
-                description = None
-                parts = [str(dtype)]
-                if description:
-                    truncated = truncate_middle(description, self.max_description_length)
-                    parts.append(f'"{truncated}"')
-                lines.append(f"- {name} ({', '.join(parts)})")
-
-            return "\n".join(lines)
-        except Exception as e:
-            return f"# {table}\n\nError fetching schema: {e}"
+    @property
+    def template_name(self) -> str:
+        return "databases/columns.md.j2"
+
+    def get_context(self, conn: BaseBackend, dataset: str, table: str) -> dict[str, Any]:
+        t = self.get_table(conn, dataset, table)
+        schema = t.schema()
+
+        columns = []
+        for name, dtype in schema.items():
+            columns.append(
+                {
+                    "name": name,
+                    "type": str(dtype),
+                    "nullable": dtype.nullable if hasattr(dtype, "nullable") else True,
+                    "description": None,  # Could be populated from metadata
+                }
+            )
+
+        return {
+            "table_name": table,
+            "dataset": dataset,
+            "columns": columns,
+            "column_count": len(columns),
+        }
 
 
 class PreviewAccessor(DataAccessor):
-    """Generates preview.md with the first N rows of data as JSONL."""
+    """Generates preview.md with the first N rows of data as JSONL.
+
+    Template variables:
+        - table_name: Name of the table
+        - dataset: Schema/dataset name
+        - rows: List of row dictionaries
+        - row_count: Number of preview rows
+        - columns: List of column info dicts
+    """
 
     def __init__(self, num_rows: int = 10):
         self.num_rows = num_rows
@@ -92,121 +152,139 @@ class PreviewAccessor(DataAccessor):
     def filename(self) -> str:
         return "preview.md"
 
-    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
-        try:
-            t = self.get_table(conn, dataset, table)
-            preview_df = t.limit(self.num_rows).execute()
-
-            lines = [
-                f"# {table} - Preview",
-                "",
-                f"**Dataset:** `{dataset}`",
-                "",
-                f"## Rows ({len(preview_df)})",
-                "",
-            ]
-
-            for _, row in preview_df.iterrows():
-                row_dict = row.to_dict()
-                # Convert non-serializable types to strings
-                for key, val in row_dict.items():
-                    if val is not None and not isinstance(val, (str, int, float, bool, list, dict)):
-                        row_dict[key] = str(val)
-                lines.append(f"- {json.dumps(row_dict)}")
-
-            return "\n".join(lines)
-        except Exception as e:
-            return f"# {table} - Preview\n\nError fetching preview: {e}"
+    @property
+    def template_name(self) -> str:
+        return "databases/preview.md.j2"
+
+    def get_context(self, conn: BaseBackend, dataset: str, table: str) -> dict[str, Any]:
+        t = self.get_table(conn, dataset, table)
+        schema = t.schema()
+        preview_df = t.limit(self.num_rows).execute()
+
+        rows = []
+        for _, row in preview_df.iterrows():
+            row_dict = row.to_dict()
+            # Convert non-serializable types to strings
+            for key, val in row_dict.items():
+                if val is not None and not isinstance(val, (str, int, float, bool, list, dict)):
+                    row_dict[key] = str(val)
+            rows.append(row_dict)
+
+        columns = [{"name": name, "type": str(dtype)} for name, dtype in schema.items()]
+
+        return {
+            "table_name": table,
+            "dataset": dataset,
+            "rows": rows,
+            "row_count": len(rows),
+            "columns": columns,
+        }
 
 
 class DescriptionAccessor(DataAccessor):
-    """Generates description.md with table metadata (row count, column count, etc.)."""
+    """Generates description.md with table metadata (row count, column count, etc.).
+
+    Template variables:
+        - table_name: Name of the table
+        - dataset: Schema/dataset name
+        - row_count: Total rows in the table
+        - column_count: Number of columns
+        - description: Table description (if available)
+        - columns: List of column info dicts
+    """
 
     @property
     def filename(self) -> str:
         return "description.md"
 
-    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
-        try:
-            t = self.get_table(conn, dataset, table)
-            schema = t.schema()
-
-            row_count = t.count().execute()
-            col_count = len(schema)
-
-            lines = [
-                f"# {table}",
-                "",
-                f"**Dataset:** `{dataset}`",
-                "",
-                "## Table Metadata",
-                "",
-                "| Property | Value |",
-                "|----------|-------|",
-                f"| **Row Count** | {row_count:,} |",
-                f"| **Column Count** | {col_count} |",
-                "",
-                "## Description",
-                "",
-                "_No description available._",
-                "",
-            ]
-
-            return "\n".join(lines)
-        except Exception as e:
-            return f"# {table}\n\nError fetching description: {e}"
+    @property
+    def template_name(self) -> str:
+        return "databases/description.md.j2"
+
+    def get_context(self, conn: BaseBackend, dataset: str, table: str) -> dict[str, Any]:
+        t = self.get_table(conn, dataset, table)
+        schema = t.schema()
+
+        row_count = t.count().execute()
+        columns = [{"name": name, "type": str(dtype)} for name, dtype in schema.items()]
+
+        return {
+            "table_name": table,
+            "dataset": dataset,
+            "row_count": row_count,
+            "column_count": len(schema),
+            "description": None,  # Could be populated from metadata
+            "columns": columns,
+        }
 
 
 class ProfilingAccessor(DataAccessor):
-    """Generates profiling.md with column statistics and data profiling."""
+    """Generates profiling.md with column statistics and data profiling.
+
+    Template variables:
+        - table_name: Name of the table
+        - dataset: Schema/dataset name
+        - column_stats: List of dicts with stats for each column:
+            - name: Column name
+            - type: Data type
+            - null_count: Number of nulls
+            - unique_count: Number of unique values
+            - min_value: Min value (numeric/temporal)
+            - max_value: Max value (numeric/temporal)
+            - error: Error message if stats couldn't be computed
+        - columns: List of column info dicts
+    """
 
     @property
     def filename(self) -> str:
         return "profiling.md"
 
-    def generate(self, conn: BaseBackend, dataset: str, table: str) -> str:
-        try:
-            t = self.get_table(conn, dataset, table)
-            schema = t.schema()
-
-            lines = [
-                f"# {table} - Profiling",
-                "",
-                f"**Dataset:** `{dataset}`",
-                "",
-                "## Column Statistics",
-                "",
-                "| Column | Type | Nulls | Unique | Min | Max |",
-                "|--------|------|-------|--------|-----|-----|",
-            ]
-
-            for name, dtype in schema.items():
-                col = t[name]
-                dtype_str = str(dtype)
-
-                try:
-                    null_count = t.filter(col.isnull()).count().execute()
-                    unique_count = col.nunique().execute()
-
-                    min_val = ""
-                    max_val = ""
-                    if dtype.is_numeric() or dtype.is_temporal():
-                        try:
-                            min_val = str(col.min().execute())
-                            max_val = str(col.max().execute())
-                            if len(min_val) > 20:
-                                min_val = min_val[:17] + "..."
-                            if len(max_val) > 20:
-                                max_val = max_val[:17] + "..."
-                        except Exception:
-                            pass
-
-                    lines.append(
-                        f"| `{name}` | `{dtype_str}` | {null_count:,} | {unique_count:,} | {min_val} | {max_val} |"
-                    )
-                except Exception as col_error:
-                    lines.append(f"| `{name}` | `{dtype_str}` | Error: {col_error} | | | |")
-
-            return "\n".join(lines)
-        except Exception as e:
-            return f"# {table} - Profiling\n\nError fetching profiling: {e}"
+    @property
+    def template_name(self) -> str:
+        return "databases/profiling.md.j2"
+
+    def get_context(self, conn: BaseBackend, dataset: str, table: str) -> dict[str, Any]:
+        t = self.get_table(conn, dataset, table)
+        schema = t.schema()
+
+        column_stats = []
+        columns = []
+
+        for name, dtype in schema.items():
+            columns.append({"name": name, "type": str(dtype)})
+            col = t[name]
+            dtype_str = str(dtype)
+
+            stat = {
+                "name": name,
+                "type": dtype_str,
+                "null_count": 0,
+                "unique_count": 0,
+                "min_value": None,
+                "max_value": None,
+                "error": None,
+            }
+
+            try:
+                stat["null_count"] = t.filter(col.isnull()).count().execute()
+                stat["unique_count"] = col.nunique().execute()
+
+                if dtype.is_numeric() or dtype.is_temporal():
+                    try:
+                        min_val = str(col.min().execute())
+                        max_val = str(col.max().execute())
+                        stat["min_value"] = truncate_middle(min_val, 20)
+                        stat["max_value"] = truncate_middle(max_val, 20)
+                    except Exception:
+                        pass
+            except Exception as col_error:
+                stat["error"] = str(col_error)
+
+            column_stats.append(stat)
+
+        return {
+            "table_name": table,
+            "dataset": dataset,
+            "column_stats": column_stats,
+            "columns": columns,
+        }

nao_core/commands/sync/cleanup.py

@@ -0,0 +1,133 @@
+"""Cleanup utilities for removing stale sync files."""
+
+import shutil
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from rich.console import Console
+
+console = Console()
+
+
+@dataclass
+class DatabaseSyncState:
+    """Tracks the state of a database sync operation.
+
+    Used to track which paths were synced so stale paths can be cleaned up.
+    """
+
+    db_path: Path
+    """The root path for this database (e.g., databases/type=duckdb/database=mydb)"""
+
+    synced_schemas: set[str] = field(default_factory=set)
+    """Set of schema names that were synced"""
+
+    synced_tables: dict[str, set[str]] = field(default_factory=dict)
+    """Dict mapping schema names to sets of table names that were synced"""
+
+    schemas_synced: int = 0
+    """Count of schemas synced"""
+
+    tables_synced: int = 0
+    """Count of tables synced"""
+
+    def add_table(self, schema: str, table: str) -> None:
+        """Record that a table was synced.
+
+        Args:
+            schema: The schema/dataset name
+            table: The table name
+        """
+        self.synced_schemas.add(schema)
+        if schema not in self.synced_tables:
+            self.synced_tables[schema] = set()
+        self.synced_tables[schema].add(table)
+        self.tables_synced += 1
+
+    def add_schema(self, schema: str) -> None:
+        """Record that a schema was synced (even if empty).
+
+        Args:
+            schema: The schema/dataset name
+        """
+        self.synced_schemas.add(schema)
+        self.schemas_synced += 1
+
+
+def cleanup_stale_paths(state: DatabaseSyncState, verbose: bool = False) -> int:
+    """Remove directories that exist on disk but weren't synced.
+
+    This function cleans up:
+    - Table directories that no longer exist in the source
+    - Schema directories that no longer exist or have no tables
+
+    Args:
+        state: The sync state tracking what was synced
+        verbose: Whether to print cleanup messages
+
+    Returns:
+        Number of stale paths removed
+    """
+    removed_count = 0
+
+    if not state.db_path.exists():
+        return 0
+
+    # Find all existing schema directories
+    existing_schemas = {
+        d.name.replace("schema=", ""): d for d in state.db_path.iterdir() if d.is_dir() and d.name.startswith("schema=")
+    }
+
+    # Remove schemas that weren't synced
+    for schema_name, schema_path in existing_schemas.items():
+        if schema_name not in state.synced_schemas:
+            if verbose:
+                console.print(f"  [dim red]removing stale schema:[/dim red] {schema_name}")
+            shutil.rmtree(schema_path)
+            removed_count += 1
+            continue
+
+        # Find existing tables in this schema
+        existing_tables = {
+            d.name.replace("table=", ""): d for d in schema_path.iterdir() if d.is_dir() and d.name.startswith("table=")
+        }
+
+        synced_tables_for_schema = state.synced_tables.get(schema_name, set())
+
+        # Remove tables that weren't synced
+        for table_name, table_path in existing_tables.items():
+            if table_name not in synced_tables_for_schema:
+                if verbose:
+                    console.print(f"  [dim red]removing stale table:[/dim red] {schema_name}.{table_name}")
+                shutil.rmtree(table_path)
+                removed_count += 1
+
+    return removed_count
+
+
+def cleanup_stale_database_types(base_path: Path, active_db_types: set[str], verbose: bool = False) -> int:
+    """Remove database type directories that are no longer configured.
+
+    Args:
+        base_path: The base databases output path
+        active_db_types: Set of database type directory names that should exist
+                         (e.g., {'type=duckdb', 'type=postgres'})
+        verbose: Whether to print cleanup messages
+
+    Returns:
+        Number of stale database type directories removed
+    """
+    removed_count = 0
+
+    if not base_path.exists():
+        return 0
+
+    for db_type_dir in base_path.iterdir():
+        if db_type_dir.is_dir() and db_type_dir.name.startswith("type="):
+            if db_type_dir.name not in active_db_types:
+                if verbose:
+                    console.print(f"  [dim red]removing stale database type:[/dim red] {db_type_dir.name}")
+                shutil.rmtree(db_type_dir)
+                removed_count += 1
+
+    return removed_count

nao_core/commands/sync/providers/__init__.py

@@ -0,0 +1,30 @@
+"""Sync providers for different resource types."""
+
+from .base import SyncProvider, SyncResult
+from .databases.provider import DatabaseSyncProvider
+from .repositories.provider import RepositorySyncProvider
+
+# Default providers in order of execution
+DEFAULT_PROVIDERS: list[SyncProvider] = [
+    RepositorySyncProvider(),
+    DatabaseSyncProvider(),
+]
+
+
+def get_all_providers() -> list[SyncProvider]:
+    """Get all registered sync providers.
+
+    Returns:
+            List of sync provider instances
+    """
+    return DEFAULT_PROVIDERS.copy()
+
+
+__all__ = [
+    "SyncProvider",
+    "SyncResult",
+    "DatabaseSyncProvider",
+    "RepositorySyncProvider",
+    "DEFAULT_PROVIDERS",
+    "get_all_providers",
+]

nao_core/commands/sync/providers/base.py

@@ -0,0 +1,87 @@
+"""Base class for sync providers."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from nao_core.config import NaoConfig
+
+
+@dataclass
+class SyncResult:
+    """Result of a sync operation."""
+
+    provider_name: str
+    items_synced: int
+    details: dict[str, Any] | None = None
+    summary: str | None = None
+
+    def get_summary(self) -> str:
+        """Get a human-readable summary of the sync result."""
+        if self.summary:
+            return self.summary
+        return f"{self.items_synced} synced"
+
+
+class SyncProvider(ABC):
+    """Abstract base class for sync providers.
+
+    A sync provider is responsible for synchronizing a specific type of resource
+    (e.g., repositories, databases) from the nao configuration to local files.
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable name for this provider (e.g., 'Repositories', 'Databases')."""
+        ...
+
+    @property
+    @abstractmethod
+    def emoji(self) -> str:
+        """Emoji icon for this provider."""
+        ...
+
+    @property
+    @abstractmethod
+    def default_output_dir(self) -> str:
+        """Default output directory for this provider."""
+        ...
+
+    @abstractmethod
+    def get_items(self, config: NaoConfig) -> list[Any]:
+        """Extract items to sync from the configuration.
+
+        Args:
+                config: The nao configuration
+
+        Returns:
+                List of items to sync (e.g., repo configs, database configs)
+        """
+        ...
+
+    @abstractmethod
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync the items to the output path.
+
+        Args:
+                items: List of items to sync
+                output_path: Path where synced data should be written
+                project_path: Path to the nao project root (for template resolution)
+
+        Returns:
+                SyncResult with statistics about what was synced
+        """
+        ...
+
+    def should_sync(self, config: NaoConfig) -> bool:
+        """Check if this provider has items to sync.
+
+        Args:
+                config: The nao configuration
+
+        Returns:
+                True if there are items to sync
+        """
+        return len(self.get_items(config)) > 0