nao-core 0.0.38__py3-none-manylinux2014_aarch64.whl

nao_core/commands/sync/providers/repositories/provider.py

@@ -0,0 +1,134 @@
+"""Repository sync provider implementation."""
+
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+
+from nao_core.commands.sync.cleanup import cleanup_stale_repos
+from nao_core.config import NaoConfig
+from nao_core.config.repos import RepoConfig
+
+from ..base import SyncProvider, SyncResult
+
+console = Console()
+
+
+def clone_or_pull_repo(repo: RepoConfig, base_path: Path) -> bool:
+    """Clone a repository if it doesn't exist, or pull latest changes if it does.
+
+    Args:
+            repo: Repository configuration
+            base_path: Base path where repositories are stored
+
+    Returns:
+            True if successful, False otherwise
+    """
+    repo_path = base_path / repo.name
+
+    try:
+        if repo_path.exists():
+            # Repository exists - pull latest changes
+            console.print(f"  [dim]Pulling latest changes for[/dim] {repo.name}")
+
+            result = subprocess.run(
+                ["git", "pull"],
+                cwd=repo_path,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+
+            if result.returncode != 0:
+                console.print(f"  [yellow]⚠[/yellow] Failed to pull {repo.name}: {result.stderr.strip()}")
+                return False
+
+            # If branch is specified, checkout that branch
+            if repo.branch:
+                subprocess.run(
+                    ["git", "checkout", repo.branch],
+                    cwd=repo_path,
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+
+        else:
+            # Repository doesn't exist - clone it
+            console.print(f"  [dim]Cloning[/dim] {repo.name}")
+
+            cmd = ["git", "clone"]
+            if repo.branch:
+                cmd.extend(["-b", repo.branch])
+            cmd.extend([repo.url, str(repo_path)])
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+
+            if result.returncode != 0:
+                console.print(f"  [yellow]⚠[/yellow] Failed to clone {repo.name}: {result.stderr.strip()}")
+                return False
+
+        return True
+
+    except Exception as e:
+        console.print(f"  [yellow]⚠[/yellow] Error syncing {repo.name}: {e}")
+        return False
+
+
+class RepositorySyncProvider(SyncProvider):
+    """Provider for syncing git repositories."""
+
+    @property
+    def name(self) -> str:
+        return "Repositories"
+
+    @property
+    def emoji(self) -> str:
+        return "📦"
+
+    @property
+    def default_output_dir(self) -> str:
+        return "repos"
+
+    def pre_sync(self, config: NaoConfig, output_path: Path) -> None:
+        """
+        Always run before syncing.
+        """
+
+        cleanup_stale_repos(config.repos, output_path, verbose=True)
+
+    def get_items(self, config: NaoConfig) -> list[RepoConfig]:
+        return config.repos
+
+    def sync(self, items: list[Any], output_path: Path, project_path: Path | None = None) -> SyncResult:
+        """Sync all configured repositories.
+
+        Args:
+                items: List of repository configurations
+                output_path: Base path where repositories are stored
+                project_path: Path to the nao project root (unused for repos)
+
+        Returns:
+                SyncResult with number of successfully synced repositories
+        """
+        if not items:
+            return SyncResult(provider_name=self.name, items_synced=0)
+
+        output_path.mkdir(parents=True, exist_ok=True)
+        success_count = 0
+
+        console.print(f"\n[bold cyan]{self.emoji} Syncing {self.name}[/bold cyan]")
+        console.print(f"[dim]Location:[/dim] {output_path.absolute()}\n")
+
+        for repo in items:
+            if clone_or_pull_repo(repo, output_path):
+                success_count += 1
+                console.print(f"  [green]✓[/green] {repo.name}")
+
+        return SyncResult(provider_name=self.name, items_synced=success_count)

nao_core/commands/sync/registry.py

@@ -0,0 +1,23 @@
+"""Accessor registry for mapping accessor types to implementations."""
+
+from nao_core.config import AccessorType
+
+from .accessors import (
+    ColumnsAccessor,
+    DataAccessor,
+    DescriptionAccessor,
+    PreviewAccessor,
+    ProfilingAccessor,
+)
+
+ACCESSOR_REGISTRY: dict[AccessorType, DataAccessor] = {
+    AccessorType.COLUMNS: ColumnsAccessor(),
+    AccessorType.PREVIEW: PreviewAccessor(num_rows=10),
+    AccessorType.DESCRIPTION: DescriptionAccessor(),
+    AccessorType.PROFILING: ProfilingAccessor(),
+}
+
+
+def get_accessors(accessor_types: list[AccessorType]) -> list[DataAccessor]:
+    """Get accessor instances for the given types."""
+    return [ACCESSOR_REGISTRY[t] for t in accessor_types if t in ACCESSOR_REGISTRY]

nao_core/config/__init__.py

@@ -0,0 +1,30 @@
+from .base import NaoConfig
+from .databases import (
+    AccessorType,
+    AnyDatabaseConfig,
+    BigQueryConfig,
+    DatabaseType,
+    DatabricksConfig,
+    DuckDBConfig,
+    PostgresConfig,
+    SnowflakeConfig,
+)
+from .exceptions import InitError
+from .llm import LLMConfig, LLMProvider
+from .slack import SlackConfig
+
+__all__ = [
+    "NaoConfig",
+    "AccessorType",
+    "AnyDatabaseConfig",
+    "BigQueryConfig",
+    "DuckDBConfig",
+    "DatabricksConfig",
+    "SnowflakeConfig",
+    "PostgresConfig",
+    "DatabaseType",
+    "LLMConfig",
+    "LLMProvider",
+    "SlackConfig",
+    "InitError",
+]

nao_core/config/base.py

@@ -0,0 +1,100 @@
+import os
+import re
+from pathlib import Path
+
+import dotenv
+import yaml
+from ibis import BaseBackend
+from pydantic import BaseModel, Field, model_validator
+
+from .databases import AnyDatabaseConfig, parse_database_config
+from .llm import LLMConfig
+from .notion import NotionConfig
+from .repos import RepoConfig
+from .slack import SlackConfig
+
+dotenv.load_dotenv()
+
+
+class NaoConfig(BaseModel):
+    """nao project configuration."""
+
+    project_name: str = Field(description="The name of the nao project")
+    databases: list[AnyDatabaseConfig] = Field(default_factory=list, description="The databases to use")
+    repos: list[RepoConfig] = Field(default_factory=list, description="The repositories to use")
+    notion: NotionConfig | None = Field(default=None, description="The Notion configurations")
+    llm: LLMConfig | None = Field(default=None, description="The LLM configuration")
+    slack: SlackConfig | None = Field(default=None, description="The Slack configuration")
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_databases(cls, data: dict) -> dict:
+        """Parse database configs into their specific types."""
+        if "databases" in data and isinstance(data["databases"], list):
+            data["databases"] = [parse_database_config(db) if isinstance(db, dict) else db for db in data["databases"]]
+        return data
+
+    def save(self, path: Path) -> None:
+        """Save the configuration to a YAML file."""
+        config_file = path / "nao_config.yaml"
+        with config_file.open("w") as f:
+            yaml.dump(
+                self.model_dump(mode="json", by_alias=True),
+                f,
+                default_flow_style=False,
+                sort_keys=False,
+                allow_unicode=True,
+            )
+
+    @classmethod
+    def load(cls, path: Path) -> "NaoConfig":
+        """Load the configuration from a YAML file."""
+        config_file = path / "nao_config.yaml"
+        content = config_file.read_text()
+        content = cls._process_env_vars(content)
+        data = yaml.safe_load(content)
+        return cls.model_validate(data)
+
+    def get_connection(self, name: str) -> BaseBackend:
+        """Get an Ibis connection by database name."""
+        for db in self.databases:
+            if db.name == name:
+                return db.connect()
+        raise ValueError(f"Database '{name}' not found in configuration")
+
+    def get_all_connections(self) -> dict[str, BaseBackend]:
+        """Get all Ibis connections as a dict keyed by name."""
+        return {db.name: db.connect() for db in self.databases}
+
+    @classmethod
+    def try_load(cls, path: Path | None = None) -> "NaoConfig | None":
+        """Try to load config from path, returns None if not found or invalid.
+
+        Args:
+            path: Directory containing nao_config.yaml. Defaults to NAO_DEFAULT_PROJECT_PATH
+                  environment variable if set, otherwise current directory.
+        """
+        if path is None:
+            default_path = os.environ.get("NAO_DEFAULT_PROJECT_PATH")
+            path = Path(default_path) if default_path else Path.cwd()
+        try:
+            os.chdir(path)
+            return cls.load(path)
+        except (FileNotFoundError, ValueError, yaml.YAMLError):
+            return None
+
+    @classmethod
+    def json_schema(cls) -> dict:
+        """Generate JSON schema for the configuration."""
+        return cls.model_json_schema()
+
+    @staticmethod
+    def _process_env_vars(content: str) -> str:
+        # Support both ${{ env('VAR') }} and {{ env('VAR') }} formats
+        regex = re.compile(r"\$?\{\{\s*env\(['\"]([^'\"]+)['\"]\)\s*\}\}")
+
+        def replacer(match: re.Match[str]) -> str:
+            env_var = match.group(1)
+            return os.environ.get(env_var, "")
+
+        return regex.sub(replacer, content)

nao_core/config/databases/__init__.py

@@ -0,0 +1,55 @@
+from typing import Annotated, Union
+
+from pydantic import Discriminator, Tag
+
+from .base import AccessorType, DatabaseConfig, DatabaseType
+from .bigquery import BigQueryConfig
+from .databricks import DatabricksConfig
+from .duckdb import DuckDBConfig
+from .postgres import PostgresConfig
+from .snowflake import SnowflakeConfig
+
+# =============================================================================
+# Database Config Registry
+# =============================================================================
+
+AnyDatabaseConfig = Annotated[
+    Union[
+        Annotated[BigQueryConfig, Tag("bigquery")],
+        Annotated[DatabricksConfig, Tag("databricks")],
+        Annotated[SnowflakeConfig, Tag("snowflake")],
+        Annotated[DuckDBConfig, Tag("duckdb")],
+        Annotated[PostgresConfig, Tag("postgres")],
+    ],
+    Discriminator("type"),
+]
+
+
+def parse_database_config(data: dict) -> DatabaseConfig:
+    """Parse a database config dict into the appropriate type."""
+    db_type = data.get("type")
+    if db_type == "bigquery":
+        return BigQueryConfig.model_validate(data)
+    elif db_type == "duckdb":
+        return DuckDBConfig.model_validate(data)
+    elif db_type == "databricks":
+        return DatabricksConfig.model_validate(data)
+    elif db_type == "snowflake":
+        return SnowflakeConfig.model_validate(data)
+    elif db_type == "postgres":
+        return PostgresConfig.model_validate(data)
+    else:
+        raise ValueError(f"Unknown database type: {db_type}")
+
+
+__all__ = [
+    "AccessorType",
+    "AnyDatabaseConfig",
+    "BigQueryConfig",
+    "DuckDBConfig",
+    "DatabaseConfig",
+    "DatabaseType",
+    "DatabricksConfig",
+    "SnowflakeConfig",
+    "PostgresConfig",
+]

nao_core/config/databases/base.py

@@ -0,0 +1,85 @@
+import fnmatch
+from abc import ABC, abstractmethod
+from enum import Enum
+
+from ibis import BaseBackend
+from pydantic import BaseModel, Field
+from rich.console import Console
+
+console = Console()
+
+
+class DatabaseType(str, Enum):
+    """Supported database types."""
+
+    BIGQUERY = "bigquery"
+    DUCKDB = "duckdb"
+    DATABRICKS = "databricks"
+    SNOWFLAKE = "snowflake"
+    POSTGRES = "postgres"
+
+
+class AccessorType(str, Enum):
+    """Available data accessors for sync."""
+
+    COLUMNS = "columns"
+    PREVIEW = "preview"
+    DESCRIPTION = "description"
+    PROFILING = "profiling"
+
+
+class DatabaseConfig(BaseModel, ABC):
+    """Base configuration for all database backends."""
+
+    name: str = Field(description="A friendly name for this connection")
+
+    # Sync settings
+    accessors: list[AccessorType] = Field(
+        default=[AccessorType.COLUMNS, AccessorType.PREVIEW, AccessorType.DESCRIPTION],
+        description="List of accessors to run during sync (columns, preview, description, profiling)",
+    )
+    include: list[str] = Field(
+        default_factory=list,
+        description="Glob patterns for schemas/tables to include (e.g., 'prod_*.*', 'analytics.dim_*'). Empty means include all.",
+    )
+    exclude: list[str] = Field(
+        default_factory=list,
+        description="Glob patterns for schemas/tables to exclude (e.g., 'temp_*.*', '*.backup_*')",
+    )
+
+    @abstractmethod
+    def connect(self) -> BaseBackend:
+        """Create an Ibis connection for this database."""
+        ...
+
+    def matches_pattern(self, schema: str, table: str) -> bool:
+        """Check if a schema.table matches the include/exclude patterns.
+
+        Args:
+            schema: The schema/dataset name
+            table: The table name
+
+        Returns:
+            True if the table should be included, False if excluded
+        """
+        full_name = f"{schema}.{table}"
+
+        # If include patterns exist, table must match at least one
+        if self.include:
+            included = any(fnmatch.fnmatch(full_name, pattern) for pattern in self.include)
+            if not included:
+                return False
+
+        # If exclude patterns exist, table must not match any
+        if self.exclude:
+            excluded = any(fnmatch.fnmatch(full_name, pattern) for pattern in self.exclude)
+            if excluded:
+                return False
+
+        return True
+
+    @abstractmethod
+    def get_database_name(self) -> str:
+        """Get the database name for this database type."""
+
+        ...

nao_core/config/databases/bigquery.py

@@ -0,0 +1,99 @@
+import json
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field, field_validator
+from rich.prompt import Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class BigQueryConfig(DatabaseConfig):
+    """BigQuery-specific configuration."""
+
+    type: Literal["bigquery"] = "bigquery"
+    project_id: str = Field(description="GCP project ID")
+    dataset_id: str | None = Field(default=None, description="Default BigQuery dataset")
+    credentials_path: str | None = Field(
+        default=None,
+        description="Path to service account JSON file. If not provided, uses Application Default Credentials (ADC)",
+    )
+    credentials_json: dict | None = Field(
+        default=None,
+        description="Service account credentials as a dict or JSON string. Takes precedence over credentials_path if both are provided",
+    )
+    sso: bool = Field(default=False, description="Use Single Sign-On (SSO) for authentication")
+    location: str | None = Field(default=None, description="BigQuery location")
+
+    @field_validator("credentials_json", mode="before")
+    @classmethod
+    def parse_credentials_json(cls, v: str | dict | None) -> dict | None:
+        if v is None:
+            return None
+        if isinstance(v, dict):
+            return v
+        if isinstance(v, str):
+            return json.loads(v)
+        raise ValueError("credentials_json must be a dict or JSON string")
+
+    @classmethod
+    def promptConfig(cls) -> "BigQueryConfig":
+        """Interactively prompt the user for BigQuery configuration."""
+        console.print("\n[bold cyan]BigQuery Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="bigquery-prod")
+
+        project_id = Prompt.ask("[bold]GCP Project ID[/bold]")
+        if not project_id:
+            raise InitError("GCP Project ID cannot be empty.")
+
+        dataset_id = Prompt.ask("[bold]Default dataset[/bold] [dim](optional, press Enter to skip)[/dim]", default="")
+
+        credentials_path = Prompt.ask(
+            "[bold]Service account JSON path[/bold] [dim](optional, uses ADC if empty)[/dim]",
+            default="",
+        )
+
+        return BigQueryConfig(
+            name=name,
+            project_id=project_id,
+            dataset_id=dataset_id or None,
+            credentials_path=credentials_path or None,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis BigQuery connection."""
+        kwargs: dict = {"project_id": self.project_id}
+
+        if self.dataset_id:
+            kwargs["dataset_id"] = self.dataset_id
+
+        if self.sso:
+            kwargs["auth_local_webserver"] = True
+
+        if self.credentials_json:
+            from google.oauth2 import service_account
+
+            credentials = service_account.Credentials.from_service_account_info(
+                self.credentials_json,
+                scopes=["https://www.googleapis.com/auth/bigquery"],
+            )
+            kwargs["credentials"] = credentials
+        elif self.credentials_path:
+            from google.oauth2 import service_account
+
+            credentials = service_account.Credentials.from_service_account_file(
+                self.credentials_path,
+                scopes=["https://www.googleapis.com/auth/bigquery"],
+            )
+            kwargs["credentials"] = credentials
+
+        return ibis.bigquery.connect(**kwargs)
+
+    def get_database_name(self) -> str:
+        """Get the database name for BigQuery."""
+
+        return self.project_id

nao_core/config/databases/databricks.py

@@ -0,0 +1,79 @@
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from nao_core.config.exceptions import InitError
+
+from .base import DatabaseConfig, console
+
+
+class DatabricksConfig(DatabaseConfig):
+    """Databricks-specific configuration."""
+
+    type: Literal["databricks"] = "databricks"
+    server_hostname: str = Field(description="Databricks server hostname (e.g., 'adb-xxxx.azuredatabricks.net')")
+    http_path: str = Field(description="HTTP path to the SQL warehouse or cluster")
+    access_token: str = Field(description="Databricks personal access token")
+    catalog: str | None = Field(default=None, description="Unity Catalog name (optional)")
+    schema_name: str | None = Field(
+        default=None,
+        validation_alias="schema",
+        serialization_alias="schema",
+        description="Default schema (optional)",
+    )
+
+    @classmethod
+    def promptConfig(cls) -> "DatabricksConfig":
+        """Interactively prompt the user for Databricks configuration."""
+        console.print("\n[bold cyan]Databricks Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="databricks-prod")
+
+        server_hostname = Prompt.ask("[bold]Server hostname[/bold] [dim](e.g., adb-xxxx.azuredatabricks.net)[/dim]")
+        if not server_hostname:
+            raise InitError("Server hostname cannot be empty.")
+
+        http_path = Prompt.ask("[bold]HTTP path[/bold] [dim](e.g., /sql/1.0/warehouses/xxxx)[/dim]")
+        if not http_path:
+            raise InitError("HTTP path cannot be empty.")
+
+        access_token = Prompt.ask("[bold]Access token[/bold]", password=True)
+        if not access_token:
+            raise InitError("Access token cannot be empty.")
+
+        catalog = Prompt.ask("[bold]Catalog[/bold] [dim](optional, press Enter to skip)[/dim]", default=None)
+
+        schema = Prompt.ask("[bold]Default schema[/bold] [dim](optional, press Enter to skip)[/dim]", default=None)
+
+        return DatabricksConfig(
+            name=name,
+            server_hostname=server_hostname,
+            http_path=http_path,
+            access_token=access_token,
+            catalog=catalog,
+            schema_name=schema,
+        )
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis Databricks connection."""
+        kwargs: dict = {
+            "server_hostname": self.server_hostname,
+            "http_path": self.http_path,
+            "access_token": self.access_token,
+        }
+
+        if self.catalog:
+            kwargs["catalog"] = self.catalog
+
+        if self.schema_name:
+            kwargs["schema"] = self.schema_name
+
+        return ibis.databricks.connect(**kwargs)
+
+    def get_database_name(self) -> str:
+        """Get the database name for Databricks."""
+
+        return self.catalog or "main"

nao_core/config/databases/duckdb.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+from typing import Literal
+
+import ibis
+from ibis import BaseBackend
+from pydantic import Field
+from rich.prompt import Prompt
+
+from .base import DatabaseConfig, console
+
+
+class DuckDBConfig(DatabaseConfig):
+    """DuckDB-specific configuration."""
+
+    type: Literal["duckdb"] = "duckdb"
+    path: str = Field(description="Path to the DuckDB database file", default=":memory:")
+
+    @classmethod
+    def promptConfig(cls) -> "DuckDBConfig":
+        """Interactively prompt the user for DuckDB configuration."""
+        console.print("\n[bold cyan]DuckDB Configuration[/bold cyan]\n")
+
+        name = Prompt.ask("[bold]Connection name[/bold]", default="duckdb-memory")
+
+        path = Prompt.ask("[bold]Path to the DuckDB database file[/bold]", default=":memory:")
+
+        return DuckDBConfig(name=name, path=path)
+
+    def connect(self) -> BaseBackend:
+        """Create an Ibis DuckDB connection."""
+        return ibis.duckdb.connect(
+            database=self.path,
+            read_only=False if self.path == ":memory:" else True,
+        )
+
+    def get_database_name(self) -> str:
+        """Get the database name for DuckDB."""
+
+        if self.path == ":memory:":
+            return "memory"
+        return Path(self.path).stem