pytrilogy 0.3.142__cp312-cp312-win_amd64.whl

trilogy/scripts/agent_info.py

@@ -0,0 +1,303 @@
+"""Agent info command - outputs AGENTS.md-style usage guide for AI agents."""
+
+from click import pass_context
+
+from trilogy.ai.prompts import get_trilogy_prompt
+
+AGENT_INFO_OUTPUT = """# Trilogy CLI - AI Agent Usage Guide
+
+## Overview
+
+Trilogy is a semantic ETL and reporting tool providing a SQL-like language with
+optimizations. This CLI enables workspace management, script execution, testing,
+and data ingestion.
+
+## Quick Start
+
+```bash
+# Initialize a new workspace
+trilogy init [path]
+
+# Run a script
+trilogy run script.preql dialect [connection_args...]
+
+# Run unit tests (mocked datasources)
+trilogy unit script.preql
+
+# Run integration tests (real connections)
+trilogy integration script.preql dialect [connection_args...]
+```
+
+## Commands Reference
+
+### trilogy init [path]
+
+Create a new Trilogy workspace with default configuration and structure.
+
+**Arguments:**
+- `path` (optional): Directory to initialize (default: current directory)
+
+**Creates:**
+- `trilogy.toml` - Configuration file
+- `raw/` - Directory for raw data models
+- `derived/` - Directory for derived data models
+- `jobs/` - Directory for job scripts
+- `hello_world.preql` - Example script
+
+**Example:**
+```bash
+trilogy init my_project
+cd my_project
+trilogy unit hello_world.preql
+```
+
+---
+
+### trilogy run <input> [dialect] [options] [conn_args...]
+
+Execute a Trilogy script or all scripts in a directory.
+
+**Arguments:**
+- `input` (required): Path to .preql file or directory
+- `dialect` (optional): Database dialect (duckdb, postgres, snowflake, bigquery, etc.)
+- `conn_args` (optional): Connection arguments passed to the database driver
+
+**Options:**
+- `--param KEY=VALUE`: Environment parameters (can be repeated)
+- `--parallelism N`, `-p N`: Max parallel workers for directory execution
+- `--config PATH`: Path to trilogy.toml configuration file
+
+**Examples:**
+```bash
+# Run single script with DuckDB
+trilogy run query.preql duckdb
+
+# Run with connection string
+trilogy run etl.preql postgres "postgresql://user:pass@host/db"
+
+# Run directory with parallelism
+trilogy run jobs/ duckdb -p 4
+
+# Run with parameters
+trilogy run report.preql duckdb --param date=2024-01-01 --param region=US
+```
+
+---
+
+### trilogy unit <input> [options]
+
+Run unit tests on Trilogy scripts with mocked datasources. Always uses DuckDB.
+
+**Arguments:**
+- `input` (required): Path to .preql file or directory
+
+**Options:**
+- `--param KEY=VALUE`: Environment parameters
+- `--parallelism N`, `-p N`: Max parallel workers
+- `--config PATH`: Path to trilogy.toml
+
+**Examples:**
+```bash
+# Test single file
+trilogy unit test_query.preql
+
+# Test entire directory
+trilogy unit tests/ -p 4
+```
+
+---
+
+### trilogy integration <input> [dialect] [options] [conn_args...]
+
+Run integration tests on Trilogy scripts with real database connections.
+
+**Arguments:**
+- `input` (required): Path to .preql file or directory
+- `dialect` (optional): Database dialect
+- `conn_args` (optional): Connection arguments
+
+**Options:**
+- `--param KEY=VALUE`: Environment parameters
+- `--parallelism N`, `-p N`: Max parallel workers
+- `--config PATH`: Path to trilogy.toml
+
+**Examples:**
+```bash
+# Integration test against Postgres
+trilogy integration tests/ postgres "postgresql://localhost/testdb"
+```
+
+---
+
+### trilogy fmt <input>
+
+Format a Trilogy script file.
+
+**Arguments:**
+- `input` (required): Path to .preql file to format
+
+**Example:**
+```bash
+trilogy fmt messy_script.preql
+```
+
+---
+
+### trilogy ingest <tables> [dialect] [options] [conn_args...]
+
+Bootstrap datasources from existing warehouse tables. Connects to a database,
+introspects table schemas, and generates Trilogy datasource definitions.
+
+**Arguments:**
+- `tables` (required): Comma-separated list of table names
+- `dialect` (optional): Database dialect
+- `conn_args` (optional): Connection arguments
+
+**Options:**
+- `--output PATH`, `-o PATH`: Output directory for generated files
+- `--schema NAME`, `-s NAME`: Schema/database to ingest from
+- `--config PATH`: Path to trilogy.toml
+- `--fks SPEC`: Foreign key relationships (format: table.col:ref_table.col)
+
+**Examples:**
+```bash
+# Ingest tables from DuckDB
+trilogy ingest "users,orders,products" duckdb "path/to/db.duckdb"
+
+# Ingest with schema and output directory
+trilogy ingest "customers" postgres -s public -o raw/ "postgresql://localhost/db"
+
+# Ingest with foreign key relationships
+trilogy ingest "orders,customers" duckdb --fks "orders.customer_id:customers.id"
+```
+
+---
+
+### trilogy serve <directory> [engine] [options]
+
+Start a FastAPI server to expose Trilogy models from a directory.
+Requires `pytrilogy[serve]` extras.
+
+**Arguments:**
+- `directory` (required): Directory containing model files
+- `engine` (optional): Engine type (default: generic)
+
+**Options:**
+- `--port N`, `-p N`: Port number (default: 8100)
+- `--host HOST`, `-h HOST`: Host to bind (default: 0.0.0.0)
+- `--timeout N`, `-t N`: Shutdown after N seconds
+
+**Endpoints exposed:**
+- `/` - Server info
+- `/index.json` - List of available models
+- `/models/<name>.json` - Specific model details
+- `/files/<name>` - Raw .preql/.sql file content
+
+**Example:**
+```bash
+trilogy serve ./models/ duckdb --port 8080
+```
+
+---
+
+### trilogy agent <command> [options]
+
+Pass off a multi-step orchestration task to an AI agent. (Not yet implemented)
+
+**Arguments:**
+- `command` (required): Natural language command
+
+**Options:**
+- `--context PATH`, `-c PATH`: Additional context files
+- `--model NAME`, `-m NAME`: AI model to use
+- `--interactive`, `-i`: Interactive mode with feedback
+
+---
+
+## Configuration File (trilogy.toml)
+
+```toml
+[engine]
+# Default dialect for execution
+dialect = "duckdb"
+
+# Max parallelism for multi-script execution
+parallelism = 3
+
+[setup]
+# Startup scripts to run before execution
+trilogy = ["setup.preql"]
+sql = ["init.sql"]
+```
+
+## Supported Dialects
+
+- `duckdb` / `duck_db` - DuckDB (default for unit tests)
+- `postgres` / `postgresql` - PostgreSQL
+- `bigquery` - Google BigQuery
+- `snowflake` - Snowflake
+- `redshift` - Amazon Redshift
+- `trino` - Trino/Presto
+- `sql_server` - Microsoft SQL Server
+
+## File Types
+
+- `.preql` - Trilogy script files (main language)
+- `.sql` - Raw SQL files (for setup scripts)
+- `trilogy.toml` - Configuration file
+
+## Common Workflows
+
+### 1. Setting up a new project
+```bash
+trilogy init my_analytics
+cd my_analytics
+# Configure trilogy.toml with your dialect and connection
+trilogy unit hello_world.preql
+```
+
+### 2. Ingesting existing tables
+```bash
+trilogy ingest "fact_sales,dim_customers,dim_products" postgres \\
+    -s analytics -o raw/ "postgresql://localhost/warehouse"
+```
+
+### 3. Running ETL jobs
+```bash
+trilogy run jobs/ postgres -p 4 "postgresql://localhost/warehouse"
+```
+
+### 4. Testing before deployment
+```bash
+# Unit tests (fast, no connection needed)
+trilogy unit .
+
+# Integration tests (real connection)
+trilogy integration . postgres "postgresql://localhost/testdb"
+```
+
+## Debug Mode
+
+Add `--debug` flag to any command for verbose output:
+```bash
+trilogy --debug run query.preql duckdb
+```
+"""
+
+
+def get_agent_info_output() -> str:
+    """Build the complete agent info output with CLI docs and syntax reference."""
+    syntax_section = get_trilogy_prompt(
+        intro="## Trilogy Language Syntax\n\nTrilogy is a SQL-inspired language with a built-in semantic layer. Use the following syntax reference when writing .preql files.",
+    )
+    return AGENT_INFO_OUTPUT + "\n" + syntax_section
+
+
+@pass_context
+def agent_info(ctx):
+    """Output comprehensive CLI documentation for AI agents.
+
+    Prints an AGENTS.md-style guide with all commands, options,
+    and usage examples optimized for AI agent consumption.
+    """
+    print(get_agent_info_output())

trilogy/scripts/common.py

@@ -0,0 +1,355 @@
+"""Common helper functions used across all CLI commands."""
+
+import traceback
+from dataclasses import dataclass
+from io import StringIO
+from pathlib import Path as PathlibPath
+from typing import Any, Iterable, Union
+
+from click.exceptions import Exit
+
+from trilogy import Executor
+from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.core.exceptions import ConfigurationException, ModelValidationError
+from trilogy.core.models.environment import Environment
+from trilogy.dialect.enums import Dialects
+from trilogy.execution.config import RuntimeConfig, load_config_file
+from trilogy.hooks.query_debugger import DebuggingHook
+from trilogy.scripts.dependency import ScriptNode
+from trilogy.scripts.display import (
+    print_error,
+    print_info,
+    print_success,
+)
+from trilogy.scripts.environment import extra_to_kwargs, parse_env_params
+
+# Configuration file name
+TRILOGY_CONFIG_NAME = "trilogy.toml"
+
+
+@dataclass
+class CLIRuntimeParams:
+    """Parameters provided via CLI for execution."""
+
+    input: str
+    dialect: Dialects | None = None
+    parallelism: int | None = None
+    param: tuple[str, ...] = ()
+    conn_args: tuple[str, ...] = ()
+    debug: bool = False
+    config_path: PathlibPath | None = None
+    execution_strategy: str = "eager_bfs"
+
+
+def merge_runtime_config(
+    cli_params: CLIRuntimeParams, file_config: RuntimeConfig
+) -> tuple[Dialects, int]:
+    """
+    Merge CLI parameters with config file settings.
+    CLI parameters take precedence over config file.
+
+    Returns:
+        tuple of (dialect, parallelism)
+
+    Raises:
+        Exit: If no dialect is specified in either CLI or config
+    """
+    # Resolve dialect: CLI argument takes precedence over config
+    if cli_params.dialect:
+        dialect = cli_params.dialect
+    elif file_config.engine_dialect:
+        dialect = file_config.engine_dialect
+    else:
+        print_error(
+            "No dialect specified. Provide dialect as argument or set engine.dialect in config file."
+        )
+        raise Exit(1)
+
+    # Resolve parallelism: CLI argument takes precedence over config
+    parallelism = (
+        cli_params.parallelism
+        if cli_params.parallelism is not None
+        else file_config.parallelism
+    )
+
+    return dialect, parallelism
+
+
+def find_trilogy_config(start_path: PathlibPath | None = None) -> PathlibPath | None:
+    """
+    Search for trilogy.toml starting from the given path, walking up parent directories.
+
+    Args:
+        start_path: Starting directory for search. If None, uses current working directory.
+
+    Returns:
+        Path to trilogy.toml if found, None otherwise.
+    """
+    search_path = start_path if start_path else PathlibPath.cwd()
+    if not search_path.is_dir():
+        search_path = search_path.parent
+
+    for parent in [search_path] + list(search_path.parents):
+        candidate = parent / TRILOGY_CONFIG_NAME
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def resolve_input(path: PathlibPath) -> list[PathlibPath]:
+    # Directory
+    if path.is_dir():
+        pattern = "**/*.preql"
+        return sorted(path.glob(pattern))
+    # Single file
+    if path.exists() and path.is_file():
+        return [path]
+
+    raise FileNotFoundError(f"Input path '{path}' does not exist.")
+
+
+def get_runtime_config(
+    path: PathlibPath, config_override: PathlibPath | None = None
+) -> RuntimeConfig:
+    config_path: PathlibPath | None = None
+
+    if config_override:
+        config_path = config_override
+    else:
+        config_path = find_trilogy_config(path)
+
+    if not config_path:
+        return RuntimeConfig(startup_trilogy=[], startup_sql=[])
+
+    try:
+        return load_config_file(config_path)
+    except Exception as e:
+        print_error(f"Failed to load configuration file {config_path}: {e}")
+        handle_execution_exception(e)
+        # This won't be reached due to handle_execution_exception raising Exit
+        return RuntimeConfig(startup_trilogy=[], startup_sql=[])
+
+
+def resolve_input_information(
+    input: str, config_path_input: PathlibPath | None = None
+) -> tuple[Iterable[PathlibPath | StringIO], PathlibPath, str, str, RuntimeConfig]:
+    input_as_path = PathlibPath(input)
+    files: Iterable[StringIO | PathlibPath]
+    if input_as_path.exists():
+        pathlib_path = input_as_path
+        files = resolve_input(pathlib_path)
+
+        if pathlib_path.is_dir():
+            directory = pathlib_path
+            input_type = "directory"
+            config = get_runtime_config(pathlib_path, config_path_input)
+
+        else:
+            directory = pathlib_path.parent
+            input_type = "file"
+            config = get_runtime_config(pathlib_path, config_path_input)
+
+        input_name = pathlib_path.name
+    else:
+        script = input
+        files = [StringIO(script)]
+        directory = PathlibPath.cwd()
+        input_type = "query"
+        input_name = "inline"
+        config = RuntimeConfig(startup_trilogy=[], startup_sql=[])
+    return files, directory, input_type, input_name, config
+
+
+def validate_required_connection_params(
+    conn_dict: dict[str, Any],
+    required_keys: list[str],
+    optional_keys: list[str],
+    dialect_name: str,
+) -> dict:
+    missing = [key for key in required_keys if key not in conn_dict]
+    extra = [
+        key
+        for key in conn_dict
+        if key not in required_keys and key not in optional_keys
+    ]
+    if missing:
+        raise ConfigurationException(
+            f"Missing required {dialect_name} connection parameters: {', '.join(missing)}"
+        )
+    if extra:
+        print(
+            f"Warning: Extra {dialect_name} connection parameters provided: {', '.join(extra)}"
+        )
+    return {
+        k: v for k, v in conn_dict.items() if k in required_keys or k in optional_keys
+    }
+
+
+def get_dialect_config(
+    edialect: Dialects, conn_dict: dict[str, Any], runtime_config: RuntimeConfig
+) -> Any:
+    """Get dialect configuration based on dialect type."""
+    conf: Union[Any, None] = None
+
+    if edialect == Dialects.DUCK_DB:
+        from trilogy.dialect.config import DuckDBConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict, [], ["path"], "DuckDB"
+        )
+        conf = DuckDBConfig(**conn_dict)
+    elif edialect == Dialects.SNOWFLAKE:
+        from trilogy.dialect.config import SnowflakeConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict, ["username", "password", "account"], [], "Snowflake"
+        )
+        conf = SnowflakeConfig(**conn_dict)
+    elif edialect == Dialects.SQL_SERVER:
+        from trilogy.dialect.config import SQLServerConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict,
+            ["host", "port", "username", "password", "database"],
+            [],
+            "SQL Server",
+        )
+        conf = SQLServerConfig(**conn_dict)
+    elif edialect == Dialects.POSTGRES:
+        from trilogy.dialect.config import PostgresConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict,
+            ["host", "port", "username", "password", "database"],
+            [],
+            "Postgres",
+        )
+        conf = PostgresConfig(**conn_dict)
+    elif edialect == Dialects.BIGQUERY:
+        from trilogy.dialect.config import BigQueryConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict, [], ["project"], "BigQuery"
+        )
+        conf = BigQueryConfig(**conn_dict)
+    elif edialect == Dialects.PRESTO:
+        from trilogy.dialect.config import PrestoConfig
+
+        conn_dict = validate_required_connection_params(
+            conn_dict,
+            ["host", "port", "username", "password", "catalog"],
+            [],
+            "Presto",
+        )
+        conf = PrestoConfig(**conn_dict)
+    if conf and runtime_config.engine_config:
+        conf = runtime_config.engine_config.merge_config(conf)
+    return conf
+
+
+def create_executor(
+    param: tuple[str, ...],
+    directory: PathlibPath,
+    conn_args: Iterable[str],
+    edialect: Dialects,
+    debug: bool,
+    config: RuntimeConfig,
+) -> Executor:
+    # Parse environment parameters from dedicated flag
+    namespace = DEFAULT_NAMESPACE
+    try:
+        env_params = parse_env_params(param)
+        from trilogy.scripts.display import show_environment_params
+
+        show_environment_params(env_params)
+    except ValueError as e:
+        print_error(str(e))
+        raise Exit(1) from e
+
+    # Parse connection arguments from remaining args
+    conn_dict = extra_to_kwargs(conn_args)
+
+    # Configure dialect
+    try:
+        conf = get_dialect_config(edialect, conn_dict, runtime_config=config)
+    except Exception as e:
+        handle_execution_exception(e)
+
+    # Create environment and set additional parameters if any exist
+    environment = Environment(working_path=str(directory), namespace=namespace)
+    if env_params:
+        environment.set_parameters(**env_params)
+
+    exec = Executor(
+        dialect=edialect,
+        engine=edialect.default_engine(conf=conf),
+        environment=environment,
+        hooks=[DebuggingHook()] if debug else [],
+    )
+    if config.startup_sql:
+        for script in config.startup_sql:
+            print_info(f"Executing startup SQL script: {script}")
+            exec.execute_file(script)
+    if config.startup_trilogy:
+        for script in config.startup_trilogy:
+            print_info(f"Executing startup Trilogy script: {script}")
+            exec.execute_file(script)
+    return exec
+
+
+def create_executor_for_script(
+    node: ScriptNode,
+    param: tuple[str, ...],
+    conn_args: Iterable[str],
+    edialect: Dialects,
+    debug: bool,
+    config: RuntimeConfig,
+) -> Executor:
+    """
+    Create an executor for a specific script node.
+
+    Each script gets its own executor with its own environment,
+    using the script's parent directory as the working path.
+    """
+    directory = node.path.parent
+    return create_executor(param, directory, conn_args, edialect, debug, config)
+
+
+def validate_datasources(
+    exec: Executor, mock: bool = False, quiet: bool = False
+) -> None:
+    """Validate datasources with consistent error handling.
+
+    Args:
+        exec: The executor instance
+        mock: If True, mock datasources before validation (for unit tests)
+        quiet: If True, suppress informational messages (for parallel execution)
+
+    Raises:
+        Exit: If validation fails
+    """
+    datasources = exec.environment.datasources.keys()
+    if not datasources:
+        if not quiet:
+            message = "unit" if mock else "integration"
+            print_success(f"No datasources found to {message} test.")
+        return
+
+    if mock:
+        exec.execute_text("mock datasources {};".format(", ".join(datasources)))
+
+    try:
+        exec.execute_text("validate datasources {};".format(", ".join(datasources)))
+    except ModelValidationError as e:
+        if not e.children:
+            print_error(f"Datasource validation failed: {e.message}")
+        for idx, child in enumerate(e.children or []):
+            print_error(f"Error {idx + 1}: {child.message}")
+        raise Exit(1) from e
+
+
+def handle_execution_exception(e: Exception, debug: bool = False) -> None:
+    print_error(f"Unexpected error: {e}")
+    if debug:
+        print_error(f"Full traceback:\n{traceback.format_exc()}")
+    raise Exit(1) from e