nao-core 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

nao_core/config/base.py

@@ -1,5 +1,8 @@
+import os
+import re
 from pathlib import Path
 
+import dotenv
 import yaml
 from ibis import BaseBackend
 from pydantic import BaseModel, Field, model_validator
@@ -9,6 +12,8 @@ from .llm import LLMConfig
 from .repos import RepoConfig
 from .slack import SlackConfig
 
+dotenv.load_dotenv()
+
 
 class NaoConfig(BaseModel):
     """nao project configuration."""
@@ -43,8 +48,9 @@ class NaoConfig(BaseModel):
     def load(cls, path: Path) -> "NaoConfig":
         """Load the configuration from a YAML file."""
         config_file = path / "nao_config.yaml"
-        with config_file.open() as f:
-            data = yaml.safe_load(f)
+        content = config_file.read_text()
+        content = cls._process_env_vars(content)
+        data = yaml.safe_load(content)
         return cls.model_validate(data)
 
     def get_connection(self, name: str) -> BaseBackend:
@@ -63,11 +69,14 @@ class NaoConfig(BaseModel):
         """Try to load config from path, returns None if not found or invalid.
 
         Args:
-            path: Directory containing nao_config.yaml. Defaults to current directory.
+            path: Directory containing nao_config.yaml. Defaults to NAO_DEFAULT_PROJECT_PATH
+                  environment variable if set, otherwise current directory.
         """
         if path is None:
-            path = Path.cwd()
+            default_path = os.environ.get("NAO_DEFAULT_PROJECT_PATH")
+            path = Path(default_path) if default_path else Path.cwd()
         try:
+            os.chdir(path)
             return cls.load(path)
         except (FileNotFoundError, ValueError, yaml.YAMLError):
             return None
@@ -76,3 +85,13 @@ class NaoConfig(BaseModel):
     def json_schema(cls) -> dict:
         """Generate JSON schema for the configuration."""
         return cls.model_json_schema()
+
+    @staticmethod
+    def _process_env_vars(content: str) -> str:
+        regex = re.compile(r"\$\{\{\s*env\(['\"]([^'\"]+)['\"]\)\s*\}\}")
+
+        def replacer(match: re.Match[str]) -> str:
+            env_var = match.group(1)
+            return os.environ.get(env_var, "")
+
+        return regex.sub(replacer, content)

nao_core/config/databases/__init__.py

@@ -1,20 +1,28 @@
+from typing import Annotated, Union
+
+from pydantic import Discriminator, Tag
+
 from .base import AccessorType, DatabaseConfig, DatabaseType
 from .bigquery import BigQueryConfig
+from .databricks import DatabricksConfig
+from .duckdb import DuckDBConfig
+from .postgres import PostgresConfig
+from .snowflake import SnowflakeConfig
 
 # =============================================================================
 # Database Config Registry
 # =============================================================================
 
-# When adding more backends, convert this to a discriminated union:
-# AnyDatabaseConfig = Annotated[
-#     Union[
-#         Annotated[BigQueryConfig, Tag("bigquery")],
-#         Annotated[PostgresConfig, Tag("postgres")],
-#     ],
-#     Discriminator(lambda x: x.get("type", "bigquery")),
-# ]
-
-AnyDatabaseConfig = BigQueryConfig
+AnyDatabaseConfig = Annotated[
+    Union[
+        Annotated[BigQueryConfig, Tag("bigquery")],
+        Annotated[DatabricksConfig, Tag("databricks")],
+        Annotated[SnowflakeConfig, Tag("snowflake")],
+        Annotated[DuckDBConfig, Tag("duckdb")],
+        Annotated[PostgresConfig, Tag("postgres")],
+    ],
+    Discriminator("type"),
+]
 
 
 def parse_database_config(data: dict) -> DatabaseConfig:
@@ -22,8 +30,26 @@ def parse_database_config(data: dict) -> DatabaseConfig:
     db_type = data.get("type")
     if db_type == "bigquery":
         return BigQueryConfig.model_validate(data)
+    elif db_type == "duckdb":
+        return DuckDBConfig.model_validate(data)
+    elif db_type == "databricks":
+        return DatabricksConfig.model_validate(data)
+    elif db_type == "snowflake":
+        return SnowflakeConfig.model_validate(data)
+    elif db_type == "postgres":
+        return PostgresConfig.model_validate(data)
     else:
         raise ValueError(f"Unknown database type: {db_type}")
 
 
-__all__ = ["AccessorType", "DatabaseConfig", "DatabaseType", "BigQueryConfig", "AnyDatabaseConfig"]
+__all__ = [
+    "AccessorType",
+    "AnyDatabaseConfig",
+    "BigQueryConfig",
+    "DuckDBConfig",
+    "DatabaseConfig",
+    "DatabaseType",
+    "DatabricksConfig",
+    "SnowflakeConfig",
+    "PostgresConfig",
+]

nao_core/config/databases/base.py

@@ -4,12 +4,19 @@ from enum import Enum
 
 from ibis import BaseBackend
 from pydantic import BaseModel, Field
+from rich.console import Console
+
+console = Console()
 
 
 class DatabaseType(str, Enum):
     """Supported database types."""
 
     BIGQUERY = "bigquery"
+    DUCKDB = "duckdb"
+    DATABRICKS = "databricks"
+    SNOWFLAKE = "snowflake"
+    POSTGRES = "postgres"
 
 
 class AccessorType(str, Enum):

nao_core/config/databases/bigquery.py

@@ -3,8 +3,11 @@ from typing import Literal
 import ibis
 from ibis import BaseBackend
 from pydantic import Field
+from rich.prompt import Prompt
 
-from .base import DatabaseConfig
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
 
 
 class BigQueryConfig(DatabaseConfig):
@@ -20,6 +23,31 @@ class BigQueryConfig(DatabaseConfig):
     sso: bool = Field(default=False, description="Use Single Sign-On (SSO) for authentication")
     location: str | None = Field(default=None, description="BigQuery location")
 
+    @classmethod
+    def promptConfig(cls) -> "BigQueryConfig":
+        """Interactively prompt the user for BigQuery configuration."""
+        console.print("\n[bold cyan]BigQuery Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="bigquery-prod")
+
+        project_id = Prompt.ask("[bold]GCP Project ID[/bold]")
+        if not project_id:
+            raise InitError("GCP Project ID cannot be empty.")
+
+        dataset_id = Prompt.ask("[bold]Default dataset[/bold] [dim](optional, press Enter to skip)[/dim]", default="")
+
+        credentials_path = Prompt.ask(
+            "[bold]Service account JSON path[/bold] [dim](optional, uses ADC if empty)[/dim]",
+            default="",
+        )
+
+        return BigQueryConfig(
+            name=name,
+            project_id=project_id,
+            dataset_id=dataset_id or None,
+            credentials_path=credentials_path or None,
+        )
+
     def connect(self) -> BaseBackend:
         """Create an Ibis BigQuery connection."""
         kwargs: dict = {"project_id": self.project_id}

nao_core/config/databases/databricks.py

@@ -0,0 +1,69 @@
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class DatabricksConfig(DatabaseConfig):
+    """Databricks-specific configuration."""
+
+    type: Literal["databricks"] = "databricks"
+    server_hostname: str = Field(description="Databricks server hostname (e.g., 'adb-xxxx.azuredatabricks.net')")
+    http_path: str = Field(description="HTTP path to the SQL warehouse or cluster")
+    access_token: str = Field(description="Databricks personal access token")
+    catalog: str | None = Field(default=None, description="Unity Catalog name (optional)")
+    schema: str | None = Field(default=None, description="Default schema (optional)")
+
+    @classmethod
+    def promptConfig(cls) -> "DatabricksConfig":
+        """Interactively prompt the user for Databricks configuration."""
+        console.print("\n[bold cyan]Databricks Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="databricks-prod")
+
+        server_hostname = Prompt.ask("[bold]Server hostname[/bold] [dim](e.g., adb-xxxx.azuredatabricks.net)[/dim]")
+        if not server_hostname:
+            raise InitError("Server hostname cannot be empty.")
+
+        http_path = Prompt.ask("[bold]HTTP path[/bold] [dim](e.g., /sql/1.0/warehouses/xxxx)[/dim]")
+        if not http_path:
+            raise InitError("HTTP path cannot be empty.")
+
+        access_token = Prompt.ask("[bold]Access token[/bold]", password=True)
+        if not access_token:
+            raise InitError("Access token cannot be empty.")
+
+        catalog = Prompt.ask("[bold]Catalog[/bold] [dim](optional, press Enter to skip)[/dim]", default=None)
+
+        schema = Prompt.ask("[bold]Default schema[/bold] [dim](optional, press Enter to skip)[/dim]", default=None)
+
+        return DatabricksConfig(
+            name=name,
+            server_hostname=server_hostname,
+            http_path=http_path,
+            access_token=access_token,
+            catalog=catalog,
+            schema=schema,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis Databricks connection."""
+        kwargs: dict = {
+            "server_hostname": self.server_hostname,
+            "http_path": self.http_path,
+            "access_token": self.access_token,
+        }
+
+        if self.catalog:
+            kwargs["catalog"] = self.catalog
+
+        if self.schema:
+            kwargs["schema"] = self.schema
+
+        return ibis.databricks.connect(**kwargs)

nao_core/config/databases/duckdb.py

@@ -0,0 +1,33 @@
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from .base import DatabaseConfig, console
+
+
+class DuckDBConfig(DatabaseConfig):
+    """DuckDB-specific configuration."""
+
+    type: Literal["duckdb"] = "duckdb"
+    path: str = Field(description="Path to the DuckDB database file", default=":memory:")
+
+    @classmethod
+    def promptConfig(cls) -> "DuckDBConfig":
+        """Interactively prompt the user for DuckDB configuration."""
+        console.print("\n[bold cyan]DuckDB Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="duckdb-memory")
+
+        path = Prompt.ask("[bold]Path to the DuckDB database file[/bold]", default=":memory:")
+
+        return DuckDBConfig(name=name, path=path)
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis DuckDB connection."""
+        return ibis.duckdb.connect(
+            database=self.path,
+            read_only=False if self.path == ":memory:" else True,
+        )

nao_core/config/databases/postgres.py

@@ -0,0 +1,78 @@
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class PostgresConfig(DatabaseConfig):
+    """PostgreSQL-specific configuration."""
+
+    type: Literal["postgres"] = "postgres"
+    host: str = Field(description="PostgreSQL host")
+    port: int = Field(default=5432, description="PostgreSQL port")
+    database: str = Field(description="Database name")
+    user: str = Field(description="Username")
+    password: str = Field(description="Password")
+    schema_name: str | None = Field(default=None, description="Default schema (optional, uses 'public' if not set)")
+
+    @classmethod
+    def promptConfig(cls) -> "PostgresConfig":
+        """Interactively prompt the user for PostgreSQL configuration."""
+        console.print("\n[bold cyan]PostgreSQL Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="postgres-prod")
+
+        host = Prompt.ask("[bold]Host[/bold]", default="localhost")
+
+        port = Prompt.ask("[bold]Port[/bold]", default="5432")
+        if not port.isdigit():
+            raise InitError("Port must be a valid integer.")
+
+        database = Prompt.ask("[bold]Database name[/bold]")
+        if not database:
+            raise InitError("Database name cannot be empty.")
+
+        user = Prompt.ask("[bold]Username[/bold]")
+        if not user:
+            raise InitError("Username cannot be empty.")
+
+        password = Prompt.ask("[bold]Password[/bold]", password=True)
+
+        schema_name = Prompt.ask(
+            "[bold]Default schema[/bold] [dim](optional, uses 'public' if empty)[/dim]",
+            default="",
+        )
+
+        return PostgresConfig(
+            name=name,
+            host=host,
+            port=int(port),
+            database=database,
+            user=user,
+            password=password,
+            schema_name=schema_name or None,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis PostgreSQL connection."""
+
+        kwargs: dict = {
+            "host": self.host,
+            "port": self.port,
+            "database": self.database,
+            "user": self.user,
+            "password": self.password,
+        }
+
+        if self.schema_name:
+            kwargs["schema"] = self.schema_name
+
+        return ibis.postgres.connect(
+            **kwargs,
+        )

nao_core/config/databases/snowflake.py

@@ -0,0 +1,115 @@
+from typing import Literal
+
+import ibis
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Confirm, Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class SnowflakeConfig(DatabaseConfig):
+    """Snowflake-specific configuration."""
+
+    type: Literal["snowflake"] = "snowflake"
+    username: str = Field(description="Snowflake username")
+    account_id: str = Field(description="Snowflake account identifier (e.g., 'xy12345.us-east-1')")
+    password: str | None = Field(default=None, description="Snowflake password")
+    database: str = Field(description="Snowflake database")
+    schema: str | None = Field(default=None, description="Snowflake schema (optional)")
+    warehouse: str | None = Field(default=None, description="Snowflake warehouse to use (optional)")
+    private_key_path: str | None = Field(
+        default=None,
+        description="Path to private key file for key-pair authentication",
+    )
+    passphrase: str | None = Field(
+        default=None,
+        description="Passphrase for the private key if it is encrypted",
+    )
+
+    @classmethod
+    def promptConfig(cls) -> "SnowflakeConfig":
+        """Interactively prompt the user for Snowflake configuration."""
+        console.print("\n[bold cyan]Snowflake Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="snowflake-prod")
+
+        username = Prompt.ask("[bold]Snowflake username[/bold]")
+        if not username:
+            raise InitError("Snowflake username cannot be empty.")
+
+        account_id = Prompt.ask("[bold]Snowflake account identifier[/bold]")
+        if not account_id:
+            raise InitError("Snowflake account identifier cannot be empty.")
+
+        database = Prompt.ask("[bold]Snowflake database[/bold]")
+        if not database:
+            raise InitError("Snowflake database cannot be empty.")
+
+        warehouse = Prompt.ask(
+            "[bold]Snowflake warehouse[/bold] [dim](optional, press Enter to skip)[/dim]", default=None
+        )
+
+        schema = Prompt.ask("[bold]Default schema[/bold] [dim](optional, press Enter to skip)[/dim]", default=None)
+
+        key_pair_auth = Confirm.ask("[bold]Use key-pair authentication for authentication?[/bold]", default=False)
+
+        if key_pair_auth:
+            private_key_path = Prompt.ask("[bold]Path to private key file[/bold]")
+            if not private_key_path:
+                raise InitError("Path to private key file cannot be empty.")
+            passphrase = Prompt.ask(
+                "[bold]Passphrase for the private key[/bold] [dim](optional, press Enter to skip)[/dim]",
+                default=None,
+                password=True,
+            )
+        else:
+            password = Prompt.ask("[bold]Snowflake password[/bold]", password=True)
+            if not password:
+                raise InitError("Snowflake password cannot be empty.")
+
+        return SnowflakeConfig(
+            name=name,
+            username=username,
+            password=password if not key_pair_auth else None,
+            account_id=account_id,
+            database=database,
+            warehouse=warehouse,
+            schema=schema,
+            private_key_path=private_key_path if key_pair_auth else None,
+            passphrase=passphrase if key_pair_auth else None,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis Snowflake connection."""
+        kwargs: dict = {"user": self.username}
+        kwargs["account"] = self.account_id
+
+        if self.database and self.schema:
+            kwargs["database"] = f"{self.database}/{self.schema}"
+        elif self.database:
+            kwargs["database"] = self.database
+
+        if self.warehouse:
+            kwargs["warehouse"] = self.warehouse
+
+        if self.private_key_path:
+            with open(self.private_key_path, "rb") as key_file:
+                private_key = serialization.load_pem_private_key(
+                    key_file.read(),
+                    password=self.passphrase.encode() if self.passphrase else None,
+                    backend=default_backend(),
+                )
+                # Convert to DER format which Snowflake expects
+                kwargs["private_key"] = private_key.private_bytes(
+                    encoding=serialization.Encoding.DER,
+                    format=serialization.PrivateFormat.PKCS8,
+                    encryption_algorithm=serialization.NoEncryption(),
+                )
+        kwargs["password"] = self.password
+
+        return ibis.snowflake.connect(**kwargs)

nao_core/config/exceptions.py

@@ -0,0 +1,7 @@
+"""Shared exceptions for nao_core."""
+
+
+class InitError(Exception):
+    """Base exception for init command errors."""
+
+    pass

nao_core/templates/__init__.py

@@ -0,0 +1,12 @@
+"""Template engine module for nao providers.
+
+This module provides a Jinja2-based templating system that allows users
+to customize the output of sync providers (databases, repos, etc.).
+
+Default templates are stored in this package and can be overridden by
+placing templates with the same name in the project's `templates/` directory.
+"""
+
+from .engine import TemplateEngine, get_template_engine
+
+__all__ = ["TemplateEngine", "get_template_engine"]

nao_core/templates/defaults/databases/columns.md.j2

@@ -0,0 +1,23 @@
+{#
+  Template: columns.md.j2
+  Description: Generates column documentation for a database table
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+        - nullable (bool): Whether the column allows nulls
+        - description (str|None): Column description if available
+    - column_count (int): Total number of columns
+#}
+# {{ table_name }}
+
+**Dataset:** `{{ dataset }}`
+
+## Columns ({{ column_count }})
+
+{% for col in columns %}
+- {{ col.name }} ({{ col.type }}{% if col.description %}, "{{ col.description | truncate_middle(256) }}"{% endif %})
+{% endfor %}

nao_core/templates/defaults/databases/description.md.j2

@@ -0,0 +1,32 @@
+{#
+  Template: description.md.j2
+  Description: Generates table metadata and description documentation
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - row_count (int): Total number of rows in the table
+    - column_count (int): Number of columns in the table
+    - description (str|None): Table description if available
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }}
+
+**Dataset:** `{{ dataset }}`
+
+## Table Metadata
+
+| Property | Value |
+|----------|-------|
+| **Row Count** | {{ "{:,}".format(row_count) }} |
+| **Column Count** | {{ column_count }} |
+
+## Description
+
+{% if description %}
+{{ description }}
+{% else %}
+_No description available._
+{% endif %}

nao_core/templates/defaults/databases/preview.md.j2

@@ -0,0 +1,22 @@
+{#
+  Template: preview.md.j2
+  Description: Generates a preview of table rows in JSONL format
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - rows (list): List of row dictionaries (first N rows of the table)
+    - row_count (int): Number of preview rows shown
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }} - Preview
+
+**Dataset:** `{{ dataset }}`
+
+## Rows ({{ row_count }})
+
+{% for row in rows %}
+- {{ row | to_json }}
+{% endfor %}

nao_core/templates/defaults/databases/profiling.md.j2

@@ -0,0 +1,34 @@
+{#
+  Template: profiling.md.j2
+  Description: Generates column-level statistics and profiling data
+  
+  Available variables:
+    - table_name (str): Name of the table
+    - dataset (str): Schema/dataset name
+    - column_stats (list): List of column statistics dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+        - null_count (int): Number of null values
+        - unique_count (int): Number of unique values
+        - min_value (str|None): Minimum value (for numeric/temporal columns)
+        - max_value (str|None): Maximum value (for numeric/temporal columns)
+        - error (str|None): Error message if stats couldn't be computed
+    - columns (list): List of column dictionaries with:
+        - name (str): Column name
+        - type (str): Data type
+#}
+# {{ table_name }} - Profiling
+
+**Dataset:** `{{ dataset }}`
+
+## Column Statistics
+
+| Column | Type | Nulls | Unique | Min | Max |
+|--------|------|-------|--------|-----|-----|
+{% for stat in column_stats %}
+{% if stat.error %}
+| `{{ stat.name }}` | `{{ stat.type }}` | Error: {{ stat.error }} | | | |
+{% else %}
+| `{{ stat.name }}` | `{{ stat.type }}` | {{ "{:,}".format(stat.null_count) }} | {{ "{:,}".format(stat.unique_count) }} | {{ stat.min_value or "" }} | {{ stat.max_value or "" }} |
+{% endif %}
+{% endfor %}