pytrilogy 0.3.148__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

trilogy/scripts/serve_helpers/file_discovery.py

@@ -0,0 +1,142 @@
+"""File discovery and processing utilities for the serve command."""
+
+from pathlib import Path
+
+
+def find_preql_files(directory_path: Path) -> list[Path]:
+    """Find all .preql files in the directory recursively.
+
+    Args:
+        directory_path: The root directory to search
+
+    Returns:
+        List of Path objects for all .preql files found
+    """
+    return list(directory_path.rglob("*.preql"))
+
+
+def find_sql_files(directory_path: Path) -> list[Path]:
+    """Find all .sql files in the directory recursively.
+
+    Args:
+        directory_path: The root directory to search
+
+    Returns:
+        List of Path objects for all .sql files found
+    """
+    return list(directory_path.rglob("*.sql"))
+
+
+def find_csv_files(directory_path: Path) -> list[Path]:
+    """Find all .csv files in the directory recursively.
+
+    Args:
+        directory_path: The root directory to search
+
+    Returns:
+        List of Path objects for all .csv files found
+    """
+    return list(directory_path.rglob("*.csv"))
+
+
+def find_trilogy_files(directory_path: Path) -> list[Path]:
+    """Find all .preql and .sql files in the directory recursively.
+
+    Args:
+        directory_path: The root directory to search
+
+    Returns:
+        List of Path objects for all .preql and .sql files found, sorted by path
+    """
+    preql_files = find_preql_files(directory_path)
+    sql_files = find_sql_files(directory_path)
+    return sorted(preql_files + sql_files)
+
+
+def find_all_model_files(directory_path: Path) -> list[Path]:
+    """Find all model files (.preql, .sql, .csv) in the directory recursively.
+
+    Args:
+        directory_path: The root directory to search
+
+    Returns:
+        List of Path objects for all model files found, sorted by path
+    """
+    preql_files = find_preql_files(directory_path)
+    sql_files = find_sql_files(directory_path)
+    csv_files = find_csv_files(directory_path)
+    return sorted(preql_files + sql_files + csv_files)
+
+
+def get_relative_model_name(preql_file: Path, directory_path: Path) -> str:
+    """Get the relative model name from a model file path.
+
+    Args:
+        preql_file: Path to the .preql, .sql, or .csv file
+        directory_path: Root directory path
+
+    Returns:
+        Relative model name with forward slashes and no extension
+    """
+    relative_path = preql_file.relative_to(directory_path)
+    return (
+        str(relative_path)
+        .replace("\\", "/")
+        .replace(".preql", "")
+        .replace(".sql", "")
+        .replace(".csv", "")
+    )
+
+
+def get_safe_model_name(model_name: str) -> str:
+    """Convert a model name to a URL-safe format.
+
+    Args:
+        model_name: The model name (may contain slashes)
+
+    Returns:
+        URL-safe model name with slashes replaced by hyphens
+    """
+    return model_name.replace("/", "-")
+
+
+def extract_description_from_file(file_path: Path) -> str:
+    """Extract description from a preql or sql file's comments.
+
+    Looks for the first comment line (starting with # or --) in the first 5 lines
+    of the file and uses it as the description.
+
+    Args:
+        file_path: Path to the .preql or .sql file
+
+    Returns:
+        Description extracted from comments or a default description
+    """
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    model_name = file_path.stem
+    default_description = f"Trilogy model: {model_name}"
+
+    first_lines = content.split("\n")[:5]
+    for line in first_lines:
+        stripped = line.strip()
+        if stripped.startswith("#"):
+            return stripped.lstrip("#").strip()
+        if stripped.startswith("--"):
+            return stripped.lstrip("-").strip()
+
+    return default_description
+
+
+def read_file_content(file_path: Path) -> str:
+    """Read and return the content of a file.
+
+    Args:
+        file_path: Path to the file
+
+    Returns:
+        File content as string
+    """
+    with open(file_path, "r") as f:
+        return f.read()

trilogy/scripts/serve_helpers/index_generation.py

@@ -0,0 +1,206 @@
+"""Index and model generation utilities for the serve command."""
+
+from pathlib import Path
+
+from trilogy.execution.config import load_config_file
+from trilogy.scripts.common import TRILOGY_CONFIG_NAME
+from trilogy.scripts.serve_helpers.file_discovery import (
+    extract_description_from_file,
+    find_all_model_files,
+    find_csv_files,
+    find_trilogy_files,
+    get_relative_model_name,
+    get_safe_model_name,
+    read_file_content,
+)
+from trilogy.scripts.serve_helpers.models import (
+    ImportFile,
+    ModelImport,
+    StoreModelIndex,
+)
+
+
+def _get_model_description(directory_path: Path, trilogy_files: list[Path]) -> str:
+    """Get model description from README.md, first file, or default.
+
+    Priority order:
+    1. README.md file in the directory
+    2. First comment from first trilogy file (alphabetically)
+    3. Default description based on directory name
+
+    Args:
+        directory_path: Root directory of the model
+        trilogy_files: List of trilogy files in the directory
+
+    Returns:
+        Description string for the model
+    """
+    # Check for README.md first
+    readme_path = directory_path / "README.md"
+    if readme_path.exists():
+        try:
+            with open(readme_path, "r", encoding="utf-8") as f:
+                content = f.read().strip()
+                # Return first non-empty line or first paragraph
+                if content:
+                    # Get first line or first paragraph (up to first blank line)
+                    lines = content.split("\n")
+                    first_line = ""
+                    for line in lines:
+                        stripped = line.strip()
+                        # Skip markdown headers
+                        if stripped and not stripped.startswith("#"):
+                            first_line = stripped
+                            break
+                        # If it's a header, use it without the hash marks
+                        elif stripped.startswith("#"):
+                            first_line = stripped.lstrip("#").strip()
+                            break
+                    if first_line:
+                        return first_line
+        except Exception:
+            pass
+
+    # Fall back to first file's description
+    if trilogy_files:
+        return extract_description_from_file(trilogy_files[0])
+
+    # Default description
+    return f"Trilogy model: {directory_path.name}"
+
+
+def generate_model_index(
+    directory_path: Path, base_url: str, engine: str
+) -> list[StoreModelIndex]:
+    """Generate model index representing directory as a single model.
+
+    Args:
+        directory_path: Root directory containing trilogy files
+        base_url: Base URL for the server (e.g., "http://localhost:8100")
+        engine: Engine type (e.g., "duckdb", "generic")
+
+    Returns:
+        List with a single StoreModelIndex for the directory model
+    """
+    model_name = directory_path.name
+    safe_name = get_safe_model_name(model_name)
+
+    return [StoreModelIndex(name=model_name, url=f"{base_url}/models/{safe_name}.json")]
+
+
+def find_model_by_name(
+    model_name: str, directory_path: Path, base_url: str, engine: str
+) -> ModelImport | None:
+    """Find and construct a ModelImport representing the directory as a single model.
+
+    Args:
+        model_name: The safe model name (directory name with slashes replaced by hyphens)
+        directory_path: Root directory containing trilogy files
+        base_url: Base URL for the server
+        engine: Engine type (e.g., "duckdb", "generic")
+
+    Returns:
+        ModelImport object if the model_name matches the directory, None otherwise
+    """
+    expected_name = get_safe_model_name(directory_path.name)
+
+    if model_name != expected_name:
+        return None
+
+    # Check for trilogy.toml config
+    config_path = directory_path / TRILOGY_CONFIG_NAME
+    setup_scripts = []
+    if config_path.exists():
+        try:
+            config = load_config_file(config_path)
+            setup_scripts = config.startup_sql + config.startup_trilogy
+        except Exception:
+            pass
+
+    # Find all trilogy files (preql and sql)
+    trilogy_files = find_trilogy_files(directory_path)
+
+    # Find CSV files separately
+    csv_files = find_csv_files(directory_path)
+
+    # Generate description
+    description = _get_model_description(directory_path, trilogy_files)
+
+    # Create components for each file
+    components = []
+
+    # Add setup scripts first with purpose="setup"
+    for setup_file in setup_scripts:
+        setup_path = (
+            setup_file if setup_file.is_absolute() else directory_path / setup_file
+        )
+        if setup_path.exists():
+            file_model_name = get_relative_model_name(setup_path, directory_path)
+            safe_file_name = get_safe_model_name(file_model_name)
+            file_ext = setup_path.suffix
+
+            components.append(
+                ImportFile(
+                    url=f"{base_url}/files/{safe_file_name}{file_ext}",
+                    name=file_model_name,
+                    alias="",
+                    type="sql" if file_ext == ".sql" else "trilogy",
+                    purpose="setup",
+                )
+            )
+
+    # Add all trilogy files (preql and sql) with purpose="source"
+    for trilogy_file in trilogy_files:
+        # Skip if already added as setup script
+        if any(
+            trilogy_file.samefile(s) if s.exists() else False for s in setup_scripts
+        ):
+            continue
+
+        file_model_name = get_relative_model_name(trilogy_file, directory_path)
+        safe_file_name = get_safe_model_name(file_model_name)
+        file_ext = trilogy_file.suffix
+
+        components.append(
+            ImportFile(
+                url=f"{base_url}/files/{safe_file_name}{file_ext}",
+                name=file_model_name,
+                alias="",
+                type="sql" if file_ext == ".sql" else "trilogy",
+                purpose="source",
+            )
+        )
+
+    # Add CSV files with purpose="data"
+    for csv_file in csv_files:
+        file_model_name = get_relative_model_name(csv_file, directory_path)
+        safe_file_name = get_safe_model_name(file_model_name)
+
+        components.append(
+            ImportFile(
+                url=f"{base_url}/files/{safe_file_name}.csv",
+                name=file_model_name,
+                alias=file_model_name,
+                type="csv",
+                purpose="data",
+            )
+        )
+
+    return ModelImport(
+        name=directory_path.name,
+        description=description,
+        engine=engine,
+        components=components,
+    )
+
+
+def find_file_content_by_name(file_name: str, directory_path: Path) -> str | None:
+
+    target_parts = Path(file_name.replace("-", "/")).parts
+
+    for model_file in find_all_model_files(directory_path):
+        relative_parts = model_file.relative_to(directory_path).parts
+        if relative_parts == target_parts:
+            return read_file_content(model_file)
+
+    return None

trilogy/scripts/serve_helpers/models.py

@@ -0,0 +1,38 @@
+"""Pydantic models for the serve command."""
+
+from pydantic import BaseModel, Field
+
+
+class ImportFile(BaseModel):
+    """Component file in a model import."""
+
+    url: str
+    name: str
+    alias: str = ""
+    purpose: str
+    type: str | None = None
+
+
+class ModelImport(BaseModel):
+    """Model import definition."""
+
+    name: str
+    engine: str
+    description: str
+    link: str = ""
+    tags: list[str] = Field(default_factory=list)
+    components: list[ImportFile]
+
+
+class StoreModelIndex(BaseModel):
+    """Individual model entry in the store index."""
+
+    name: str
+    url: str
+
+
+class StoreIndex(BaseModel):
+    """Store index containing list of available models."""
+
+    name: str
+    models: list[StoreModelIndex]

trilogy/scripts/single_execution.py

@@ -0,0 +1,131 @@
+import traceback
+from datetime import datetime
+from typing import Any, Union
+
+from trilogy import Executor
+from trilogy.core.statements.execute import PROCESSED_STATEMENT_TYPES
+from trilogy.scripts.display import (
+    FETCH_LIMIT,
+    ResultSet,
+    create_progress_context,
+    print_error,
+    print_info,
+    print_results_table,
+    show_statement_result,
+    show_statement_type,
+)
+
+
+def get_statement_type(statement: PROCESSED_STATEMENT_TYPES) -> str:
+    """Get the type/class name of a statement."""
+    return type(statement).__name__
+
+
+def execute_single_statement(
+    exec: Executor,
+    query: PROCESSED_STATEMENT_TYPES,
+    idx: int,
+    total_queries: int,
+    use_progress=False,
+) -> tuple[bool, ResultSet | None, Any, Union[Exception, None]]:
+    """Execute a single statement and handle results/errors consistently."""
+    # Log the statement type before execution
+    statement_type = get_statement_type(query)
+    if not use_progress:  # Only show type when not using progress bar
+        show_statement_type(idx, total_queries, statement_type)
+
+    start_time = datetime.now()
+
+    try:
+        raw_results = exec.execute_statement(query)
+        results = (
+            ResultSet(
+                rows=raw_results.fetchmany(FETCH_LIMIT + 1), columns=raw_results.keys()
+            )
+            if raw_results
+            else None
+        )
+        duration = datetime.now() - start_time
+
+        if not use_progress:
+            show_statement_result(idx, total_queries, duration, bool(results))
+
+        return True, results, duration, None
+
+    except Exception as e:
+        duration = datetime.now() - start_time
+
+        if not use_progress:
+            show_statement_result(idx, total_queries, duration, False, str(e), type(e))
+
+        return False, None, duration, e
+
+
+def execute_queries_with_progress(
+    exec: Executor, queries: list[PROCESSED_STATEMENT_TYPES]
+) -> Exception | None:
+    """Execute queries with Rich progress bar. Returns True if all succeeded, False if any failed."""
+    progress = create_progress_context()
+    results_to_print = []
+    exception = None
+
+    with progress:
+        task = progress.add_task("Executing statements...", total=len(queries))
+
+        for idx, query in enumerate(queries):
+            statement_type = get_statement_type(query)
+            progress.update(
+                task, description=f"Statement {idx+1}/{len(queries)} ({statement_type})"
+            )
+
+            success, results, duration, error = execute_single_statement(
+                exec, query, idx, len(queries), use_progress=True
+            )
+
+            if not success:
+                exception = error
+
+            # Store results for printing after progress is done
+            results_to_print.append(
+                (idx, len(queries), duration, success, results, error)
+            )
+            progress.advance(task)
+            if exception:
+                break
+
+    # Print all results after progress bar is finished
+    for idx, total_queries, duration, success, results, error in results_to_print:
+        if error:
+            show_statement_result(
+                idx, total_queries, duration, False, str(error), type(error)
+            )
+            print_error(f"Full traceback:\n{traceback.format_exc()}")
+        else:
+            show_statement_result(idx, total_queries, duration, bool(results))
+            if results and not error:
+                print_results_table(results)
+
+    return exception
+
+
+def execute_queries_simple(
+    exec: Executor, queries: list[PROCESSED_STATEMENT_TYPES]
+) -> Exception | None:
+    """Execute queries with simple output. Returns True if all succeeded, False if any failed."""
+    exception = None
+
+    for idx, query in enumerate(queries):
+        if len(queries) > 1:
+            print_info(f"Executing statement {idx+1} of {len(queries)}...")
+
+        success, results, duration, error = execute_single_statement(
+            exec, query, idx, len(queries), use_progress=False
+        )
+
+        if not success:
+            exception = error
+
+        if results and not error:
+            print_results_table(results)
+
+    return exception

trilogy/scripts/testing.py

@@ -0,0 +1,129 @@
+"""Testing commands (integration and unit) for Trilogy CLI."""
+
+from pathlib import Path as PathlibPath
+
+from click import UNPROCESSED, Path, argument, option, pass_context
+from click.exceptions import Exit
+
+from trilogy import Executor
+from trilogy.dialect.enums import Dialects
+from trilogy.scripts.common import (
+    CLIRuntimeParams,
+    ExecutionStats,
+    count_statement_stats,
+    handle_execution_exception,
+    validate_datasources,
+)
+from trilogy.scripts.dependency import ScriptNode
+from trilogy.scripts.parallel_execution import ExecutionMode, run_parallel_execution
+
+
+def execute_script_for_integration(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> ExecutionStats:
+    """Execute a script for the 'integration' command (parse + validate)."""
+    with open(node.path, "r") as f:
+        queries = exec.parse_text(f.read())
+    stats = count_statement_stats(queries)
+    validate_datasources(exec, mock=False, quiet=quiet)
+    # Count datasources validated
+    stats.validate_count = len(exec.environment.datasources)
+    return stats
+
+
+def execute_script_for_unit(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> ExecutionStats:
+    """Execute a script for the 'unit' command (parse + mock validate)."""
+    with open(node.path, "r") as f:
+        queries = exec.parse_text(f.read())
+    stats = count_statement_stats(queries)
+    validate_datasources(exec, mock=True, quiet=quiet)
+    # Count datasources validated
+    stats.validate_count = len(exec.environment.datasources)
+    return stats
+
+
+@argument("input", type=Path())
+@argument("dialect", type=str, required=False)
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@argument("conn_args", nargs=-1, type=UNPROCESSED)
+@pass_context
+def integration(
+    ctx, input, dialect: str | None, param, parallelism: int | None, config, conn_args
+):
+    """Run integration tests on Trilogy scripts."""
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects(dialect) if dialect else None,
+        parallelism=parallelism,
+        param=param,
+        conn_args=conn_args,
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_integration,
+            execution_mode=ExecutionMode.INTEGRATION,
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)
+
+
+@argument("input", type=Path())
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@pass_context
+def unit(
+    ctx,
+    input,
+    param,
+    parallelism: int | None,
+    config,
+):
+    """Run unit tests on Trilogy scripts with mocked datasources."""
+    # Build CLI runtime params (unit tests always use DuckDB)
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects.DUCK_DB,
+        parallelism=parallelism,
+        param=param,
+        conn_args=(),
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_unit,
+            execution_mode=ExecutionMode.UNIT,
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)