nao-core 0.0.30__py3-none-any.whl → 0.0.32__py3-none-any.whl

nao_core/commands/sync/providers/databases/provider.py

@@ -0,0 +1,123 @@
+"""Database sync provider implementation."""
+
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState, cleanup_stale_paths
+from nao_core.commands.sync.registry import get_accessors
+from nao_core.config import AnyDatabaseConfig, NaoConfig
+
+from ..base import SyncProvider, SyncResult
+from .bigquery import sync_bigquery
+from .databricks import sync_databricks
+from .duckdb import sync_duckdb
+from .postgres import sync_postgres
+from .snowflake import sync_snowflake
+
+console = Console()
+
+# Registry mapping database types to their sync functions
+DATABASE_SYNC_FUNCTIONS = {
+    "bigquery": sync_bigquery,
+    "duckdb": sync_duckdb,
+    "databricks": sync_databricks,
+    "snowflake": sync_snowflake,
+    "postgres": sync_postgres,
+}
+
+
+class DatabaseSyncProvider(SyncProvider):
+    """Provider for syncing database schemas to markdown documentation."""
+
+    @property
+    def name(self) -> str:
+        return "Databases"
+
+    @property
+    def emoji(self) -> str:
+        return "🗄️"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "databases"
+
+    def get_items(self, config: NaoConfig) -> list[AnyDatabaseConfig]:
+        return config.databases
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured databases.
+
+        Args:
+                items: List of database configurations
+                output_path: Base path where database schemas are stored
+                project_path: Path to the nao project root (for template resolution)
+
+        Returns:
+                SyncResult with datasets and tables synced
+        """
+        if not items:
+            console.print("\n[dim]No databases configured[/dim]")
+            return SyncResult(provider_name=self.name, items_synced=0)
+
+        # Set project path for template resolution
+        DataAccessor.set_project_path(project_path)
+
+        total_datasets = 0
+        total_tables = 0
+        total_removed = 0
+        sync_states: list[DatabaseSyncState] = []
+
+        console.print(f"\n[bold cyan]{self.emoji}  Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
+
+        with Progress(
+            SpinnerColumn(style="dim"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=30, style="dim", complete_style="cyan", finished_style="green"),
+            TaskProgressColumn(),
+            console=console,
+            transient=False,
+        ) as progress:
+            for db in items:
+                # Get accessors from database config
+                db_accessors = get_accessors(db.accessors)
+                accessor_names = [a.filename.replace(".md", "") for a in db_accessors]
+
+                try:
+                    console.print(f"[dim]{db.name} accessors:[/dim] {', '.join(accessor_names)}")
+
+                    sync_fn = DATABASE_SYNC_FUNCTIONS.get(db.type)
+                    if sync_fn:
+                        state = sync_fn(db, output_path, progress, db_accessors)
+                        sync_states.append(state)
+                        total_datasets += state.schemas_synced
+                        total_tables += state.tables_synced
+                    else:
+                        console.print(f"[yellow]⚠ Unsupported database type: {db.type}[/yellow]")
+                except Exception as e:
+                    console.print(f"[bold red]✗[/bold red] Failed to sync {db.name}: {e}")
+
+        # Clean up stale files after all syncs complete
+        for state in sync_states:
+            removed = cleanup_stale_paths(state, verbose=True)
+            total_removed += removed
+
+        # Build summary
+        summary = f"{total_tables} tables across {total_datasets} datasets"
+        if total_removed > 0:
+            summary += f", {total_removed} stale removed"
+
+        return SyncResult(
+            provider_name=self.name,
+            items_synced=total_tables,
+            details={
+                "datasets": total_datasets,
+                "tables": total_tables,
+                "removed": total_removed,
+            },
+            summary=summary,
+        )

nao_core/commands/sync/providers/databases/snowflake.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+
+from rich.progress import Progress
+
+from nao_core.commands.sync.accessors import DataAccessor
+from nao_core.commands.sync.cleanup import DatabaseSyncState
+
+
+def sync_snowflake(
+    db_config,
+    base_path: Path,
+    progress: Progress,
+    accessors: list[DataAccessor],
+) -> DatabaseSyncState:
+    """Sync Snowflake database schema to markdown files.
+
+    Args:
+            db_config: The database configuration
+            base_path: Base output path
+            progress: Rich progress instance
+            accessors: List of data accessors to run
+
+    Returns:
+            DatabaseSyncState with sync results and tracked paths
+    """
+    conn = db_config.connect()
+    db_path = base_path / "type=snowflake" / f"database={db_config.database}"
+    state = DatabaseSyncState(db_path=db_path)
+
+    if db_config.schema:
+        schemas = [db_config.schema]
+    else:
+        schemas = conn.list_databases()
+
+    schema_task = progress.add_task(
+        f"[dim]{db_config.name}[/dim]",
+        total=len(schemas),
+    )
+
+    for schema in schemas:
+        try:
+            all_tables = conn.list_tables(database=schema)
+        except Exception:
+            progress.update(schema_task, advance=1)
+            continue
+
+        # Filter tables based on include/exclude patterns
+        tables = [t for t in all_tables if db_config.matches_pattern(schema, t)]
+
+        # Skip schema if no tables match
+        if not tables:
+            progress.update(schema_task, advance=1)
+            continue
+
+        schema_path = db_path / f"schema={schema}"
+        schema_path.mkdir(parents=True, exist_ok=True)
+        state.add_schema(schema)
+
+        table_task = progress.add_task(
+            f"  [cyan]{schema}[/cyan]",
+            total=len(tables),
+        )
+
+        for table in tables:
+            table_path = schema_path / f"table={table}"
+            table_path.mkdir(parents=True, exist_ok=True)
+
+            for accessor in accessors:
+                content = accessor.generate(conn, schema, table)
+                output_file = table_path / accessor.filename
+                output_file.write_text(content)
+
+            state.add_table(schema, table)
+            progress.update(table_task, advance=1)
+
+        progress.update(schema_task, advance=1)
+
+    return state

nao_core/commands/sync/providers/repositories/__init__.py

@@ -0,0 +1,5 @@
+"""Repository syncing functionality for cloning and pulling git repositories."""
+
+from .provider import RepositorySyncProvider
+
+__all__ = ["RepositorySyncProvider"]

nao_core/commands/sync/{repositories.py → providers/repositories/provider.py}

@@ -1,12 +1,16 @@
-"""Repository syncing functionality for cloning and pulling git repositories."""
+"""Repository sync provider implementation."""
 
 import subprocess
 from pathlib import Path
+from typing import Any
 
 from rich.console import Console
 
+from nao_core.config import NaoConfig
 from nao_core.config.repos import RepoConfig
 
+from ..base import SyncProvider, SyncResult
+
 console = Console()
 
 
@@ -76,28 +80,47 @@ def clone_or_pull_repo(repo: RepoConfig, base_path: Path) -> bool:
         return False
 
 
-def sync_repositories(repos: list[RepoConfig], base_path: Path) -> int:
-    """Sync all configured repositories.
+class RepositorySyncProvider(SyncProvider):
+    """Provider for syncing git repositories."""
 
-    Args:
-            repos: List of repository configurations
-            base_path: Base path where repositories are stored
+    @property
+    def name(self) -> str:
+        return "Repositories"
 
-    Returns:
-            Number of successfully synced repositories
-    """
-    if not repos:
-        return 0
+    @property
+    def emoji(self) -> str:
+        return "📦"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "repos"
+
+    def get_items(self, config: NaoConfig) -> list[RepoConfig]:
+        return config.repos
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured repositories.
+
+        Args:
+                items: List of repository configurations
+                output_path: Base path where repositories are stored
+                project_path: Path to the nao project root (unused for repos)
+
+        Returns:
+                SyncResult with number of successfully synced repositories
+        """
+        if not items:
+            return SyncResult(provider_name=self.name, items_synced=0)
 
-    base_path.mkdir(parents=True, exist_ok=True)
-    success_count = 0
+        output_path.mkdir(parents=True, exist_ok=True)
+        success_count = 0
 
-    console.print("\n[bold cyan]📦 Syncing Repositories[/bold cyan]")
-    console.print(f"[dim]Location:[/dim] {base_path.absolute()}\n")
+        console.print(f"\n[bold cyan]{self.emoji} Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
 
-    for repo in repos:
-        if clone_or_pull_repo(repo, base_path):
-            success_count += 1
-            console.print(f"  [green]✓[/green] {repo.name}")
+        for repo in items:
+            if clone_or_pull_repo(repo, output_path):
+                success_count += 1
+                console.print(f"  [green]✓[/green] {repo.name}")
 
-    return success_count
+        return SyncResult(provider_name=self.name, items_synced=success_count)

nao_core/config/__init__.py

@@ -6,6 +6,7 @@ from .databases import (
     DatabaseType,
     DatabricksConfig,
     DuckDBConfig,
+    PostgresConfig,
     SnowflakeConfig,
 )
 from .exceptions import InitError
@@ -20,6 +21,7 @@ __all__ = [
     "DuckDBConfig",
     "DatabricksConfig",
     "SnowflakeConfig",
+    "PostgresConfig",
     "DatabaseType",
     "LLMConfig",
     "LLMProvider",

nao_core/config/base.py

@@ -1,5 +1,8 @@
+import os
+import re
 from pathlib import Path
 
+import dotenv
 import yaml
 from ibis import BaseBackend
 from pydantic import BaseModel, Field, model_validator
@@ -9,6 +12,8 @@ from .llm import LLMConfig
 from .repos import RepoConfig
 from .slack import SlackConfig
 
+dotenv.load_dotenv()
+
 
 class NaoConfig(BaseModel):
     """nao project configuration."""
@@ -43,8 +48,9 @@ class NaoConfig(BaseModel):
     def load(cls, path: Path) -> "NaoConfig":
         """Load the configuration from a YAML file."""
         config_file = path / "nao_config.yaml"
-        with config_file.open() as f:
-            data = yaml.safe_load(f)
+        content = config_file.read_text()
+        content = cls._process_env_vars(content)
+        data = yaml.safe_load(content)
         return cls.model_validate(data)
 
     def get_connection(self, name: str) -> BaseBackend:
@@ -63,11 +69,14 @@ class NaoConfig(BaseModel):
         """Try to load config from path, returns None if not found or invalid.
 
         Args:
-            path: Directory containing nao_config.yaml. Defaults to current directory.
+            path: Directory containing nao_config.yaml. Defaults to NAO_DEFAULT_PROJECT_PATH
+                  environment variable if set, otherwise current directory.
         """
         if path is None:
-            path = Path.cwd()
+            default_path = os.environ.get("NAO_DEFAULT_PROJECT_PATH")
+            path = Path(default_path) if default_path else Path.cwd()
         try:
+            os.chdir(path)
             return cls.load(path)
         except (FileNotFoundError, ValueError, yaml.YAMLError):
             return None
@@ -76,3 +85,13 @@ class NaoConfig(BaseModel):
     def json_schema(cls) -> dict:
         """Generate JSON schema for the configuration."""
         return cls.model_json_schema()
+
+    @staticmethod
+    def _process_env_vars(content: str) -> str:
+        regex = re.compile(r"\$\{\{\s*env\(['\"]([^'\"]+)['\"]\)\s*\}\}")
+
+        def replacer(match: re.Match[str]) -> str:
+            env_var = match.group(1)
+            return os.environ.get(env_var, "")
+
+        return regex.sub(replacer, content)

nao_core/config/databases/__init__.py

@@ -6,6 +6,7 @@ from .base import AccessorType, DatabaseConfig, DatabaseType
 from .bigquery import BigQueryConfig
 from .databricks import DatabricksConfig
 from .duckdb import DuckDBConfig
+from .postgres import PostgresConfig
 from .snowflake import SnowflakeConfig
 
 # =============================================================================
@@ -18,6 +19,7 @@ AnyDatabaseConfig = Annotated[
         Annotated[DatabricksConfig, Tag("databricks")],
         Annotated[SnowflakeConfig, Tag("snowflake")],
         Annotated[DuckDBConfig, Tag("duckdb")],
+        Annotated[PostgresConfig, Tag("postgres")],
     ],
     Discriminator("type"),
 ]
@@ -34,6 +36,8 @@ def parse_database_config(data: dict) -> DatabaseConfig:
         return DatabricksConfig.model_validate(data)
     elif db_type == "snowflake":
         return SnowflakeConfig.model_validate(data)
+    elif db_type == "postgres":
+        return PostgresConfig.model_validate(data)
     else:
         raise ValueError(f"Unknown database type: {db_type}")
 
@@ -47,4 +51,5 @@ __all__ = [
     "DatabaseType",
     "DatabricksConfig",
     "SnowflakeConfig",
+    "PostgresConfig",
 ]

nao_core/config/databases/base.py

@@ -16,6 +16,7 @@ class DatabaseType(str, Enum):
     DUCKDB = "duckdb"
     DATABRICKS = "databricks"
     SNOWFLAKE = "snowflake"
+    POSTGRES = "postgres"
 
 
 class AccessorType(str, Enum):

nao_core/config/databases/postgres.py

@@ -0,0 +1,78 @@
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class PostgresConfig(DatabaseConfig):
+    """PostgreSQL-specific configuration."""
+
+    type: Literal["postgres"] = "postgres"
+    host: str = Field(description="PostgreSQL host")
+    port: int = Field(default=5432, description="PostgreSQL port")
+    database: str = Field(description="Database name")
+    user: str = Field(description="Username")
+    password: str = Field(description="Password")
+    schema_name: str | None = Field(default=None, description="Default schema (optional, uses 'public' if not set)")
+
+    @classmethod
+    def promptConfig(cls) -> "PostgresConfig":
+        """Interactively prompt the user for PostgreSQL configuration."""
+        console.print("\n[bold cyan]PostgreSQL Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="postgres-prod")
+
+        host = Prompt.ask("[bold]Host[/bold]", default="localhost")
+
+        port = Prompt.ask("[bold]Port[/bold]", default="5432")
+        if not port.isdigit():
+            raise InitError("Port must be a valid integer.")
+
+        database = Prompt.ask("[bold]Database name[/bold]")
+        if not database:
+            raise InitError("Database name cannot be empty.")
+
+        user = Prompt.ask("[bold]Username[/bold]")
+        if not user:
+            raise InitError("Username cannot be empty.")
+
+        password = Prompt.ask("[bold]Password[/bold]", password=True)
+
+        schema_name = Prompt.ask(
+            "[bold]Default schema[/bold] [dim](optional, uses 'public' if empty)[/dim]",
+            default="",
+        )
+
+        return PostgresConfig(
+            name=name,
+            host=host,
+            port=int(port),
+            database=database,
+            user=user,
+            password=password,
+            schema_name=schema_name or None,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis PostgreSQL connection."""
+
+        kwargs: dict = {
+            "host": self.host,
+            "port": self.port,
+            "database": self.database,
+            "user": self.user,
+            "password": self.password,
+        }
+
+        if self.schema_name:
+            kwargs["schema"] = self.schema_name
+
+        return ibis.postgres.connect(
+            **kwargs,
+        )

nao_core/templates/__init__.py

@@ -0,0 +1,12 @@
+"""Template engine module for nao providers.
+
+This module provides a Jinja2-based templating system that allows users
+to customize the output of sync providers (databases, repos, etc.).
+
+Default templates are stored in this package and can be overridden by
+placing templates with the same name in the project's `templates/` directory.
+"""
+
+from .engine import TemplateEngine, get_template_engine
+
+__all__ = ["TemplateEngine", "get_template_engine"]

nao_core/templates/defaults/databases/columns.md.j2

@@ -0,0 +1,23 @@
+{#
+  Template: columns.md.j2
+  Description: Generates column documentation for a database table
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+        - nullable (bool): Whether the column allows nulls
+        - description (str|None): Column description if available
+    - column_count (int): Total number of columns
+#}
+# {{ table_name }}
+
+**Dataset:** `{{ dataset }}`
+
+## Columns ({{ column_count }})
+
+{% for col in columns %}
+- {{ col.name }} ({{ col.type }}{% if col.description %}, "{{ col.description | truncate_middle(256) }}"{% endif %})
+{% endfor %}

nao_core/templates/defaults/databases/description.md.j2

@@ -0,0 +1,32 @@
+{#
+  Template: description.md.j2
+  Description: Generates table metadata and description documentation
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - row_count (int): Total number of rows in the table
+    - column_count (int): Number of columns in the table
+    - description (str|None): Table description if available
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }}
+
+**Dataset:** `{{ dataset }}`
+
+## Table Metadata
+
+| Property | Value |
+|----------|-------|
+| **Row Count** | {{ "{:,}".format(row_count) }} |
+| **Column Count** | {{ column_count }} |
+
+## Description
+
+{% if description %}
+{{ description }}
+{% else %}
+_No description available._
+{% endif %}

nao_core/templates/defaults/databases/preview.md.j2

@@ -0,0 +1,22 @@
+{#
+  Template: preview.md.j2
+  Description: Generates a preview of table rows in JSONL format
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - rows (list): List of row dictionaries (first N rows of the table)
+    - row_count (int): Number of preview rows shown
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }} - Preview
+
+**Dataset:** `{{ dataset }}`
+
+## Rows ({{ row_count }})
+
+{% for row in rows %}
+- {{ row | to_json }}
+{% endfor %}

nao_core/templates/defaults/databases/profiling.md.j2

@@ -0,0 +1,34 @@
+{#
+  Template: profiling.md.j2
+  Description: Generates column-level statistics and profiling data
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - column_stats (list): List of column statistics dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+        - null_count (int): Number of null values
+        - unique_count (int): Number of unique values
+        - min_value (str|None): Minimum value (for numeric/temporal columns)
+        - max_value (str|None): Maximum value (for numeric/temporal columns)
+        - error (str|None): Error message if stats couldn't be computed
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }} - Profiling
+
+**Dataset:** `{{ dataset }}`
+
+## Column Statistics
+
+| Column | Type | Nulls | Unique | Min | Max |
+|--------|------|-------|--------|-----|-----|
+{% for stat in column_stats %}
+{% if stat.error %}
+| `{{ stat.name }}` | `{{ stat.type }}` | Error: {{ stat.error }} | | | |
+{% else %}
+| `{{ stat.name }}` | `{{ stat.type }}` | {{ "{:,}".format(stat.null_count) }} | {{ "{:,}".format(stat.unique_count) }} | {{ stat.min_value or "" }} | {{ stat.max_value or "" }} |
+{% endif %}
+{% endfor %}