pytrilogy 0.3.142__cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/scripts/serve_helpers/index_generation.py

@@ -0,0 +1,206 @@
+"""Index and model generation utilities for the serve command."""
+
+from pathlib import Path
+
+from trilogy.execution.config import load_config_file
+from trilogy.scripts.common import TRILOGY_CONFIG_NAME
+from trilogy.scripts.serve_helpers.file_discovery import (
+    extract_description_from_file,
+    find_all_model_files,
+    find_csv_files,
+    find_trilogy_files,
+    get_relative_model_name,
+    get_safe_model_name,
+    read_file_content,
+)
+from trilogy.scripts.serve_helpers.models import (
+    ImportFile,
+    ModelImport,
+    StoreModelIndex,
+)
+
+
+def _get_model_description(directory_path: Path, trilogy_files: list[Path]) -> str:
+    """Get model description from README.md, first file, or default.
+
+    Priority order:
+    1. README.md file in the directory
+    2. First comment from first trilogy file (alphabetically)
+    3. Default description based on directory name
+
+    Args:
+        directory_path: Root directory of the model
+        trilogy_files: List of trilogy files in the directory
+
+    Returns:
+        Description string for the model
+    """
+    # Check for README.md first
+    readme_path = directory_path / "README.md"
+    if readme_path.exists():
+        try:
+            with open(readme_path, "r", encoding="utf-8") as f:
+                content = f.read().strip()
+                # Return first non-empty line or first paragraph
+                if content:
+                    # Get first line or first paragraph (up to first blank line)
+                    lines = content.split("\n")
+                    first_line = ""
+                    for line in lines:
+                        stripped = line.strip()
+                        # Skip markdown headers
+                        if stripped and not stripped.startswith("#"):
+                            first_line = stripped
+                            break
+                        # If it's a header, use it without the hash marks
+                        elif stripped.startswith("#"):
+                            first_line = stripped.lstrip("#").strip()
+                            break
+                    if first_line:
+                        return first_line
+        except Exception:
+            pass
+
+    # Fall back to first file's description
+    if trilogy_files:
+        return extract_description_from_file(trilogy_files[0])
+
+    # Default description
+    return f"Trilogy model: {directory_path.name}"
+
+
+def generate_model_index(
+    directory_path: Path, base_url: str, engine: str
+) -> list[StoreModelIndex]:
+    """Generate model index representing directory as a single model.
+
+    Args:
+        directory_path: Root directory containing trilogy files
+        base_url: Base URL for the server (e.g., "http://localhost:8100")
+        engine: Engine type (e.g., "duckdb", "generic")
+
+    Returns:
+        List with a single StoreModelIndex for the directory model
+    """
+    model_name = directory_path.name
+    safe_name = get_safe_model_name(model_name)
+
+    return [StoreModelIndex(name=model_name, url=f"{base_url}/models/{safe_name}.json")]
+
+
+def find_model_by_name(
+    model_name: str, directory_path: Path, base_url: str, engine: str
+) -> ModelImport | None:
+    """Find and construct a ModelImport representing the directory as a single model.
+
+    Args:
+        model_name: The safe model name (directory name with slashes replaced by hyphens)
+        directory_path: Root directory containing trilogy files
+        base_url: Base URL for the server
+        engine: Engine type (e.g., "duckdb", "generic")
+
+    Returns:
+        ModelImport object if the model_name matches the directory, None otherwise
+    """
+    expected_name = get_safe_model_name(directory_path.name)
+
+    if model_name != expected_name:
+        return None
+
+    # Check for trilogy.toml config
+    config_path = directory_path / TRILOGY_CONFIG_NAME
+    setup_scripts = []
+    if config_path.exists():
+        try:
+            config = load_config_file(config_path)
+            setup_scripts = config.startup_sql + config.startup_trilogy
+        except Exception:
+            pass
+
+    # Find all trilogy files (preql and sql)
+    trilogy_files = find_trilogy_files(directory_path)
+
+    # Find CSV files separately
+    csv_files = find_csv_files(directory_path)
+
+    # Generate description
+    description = _get_model_description(directory_path, trilogy_files)
+
+    # Create components for each file
+    components = []
+
+    # Add setup scripts first with purpose="setup"
+    for setup_file in setup_scripts:
+        setup_path = (
+            setup_file if setup_file.is_absolute() else directory_path / setup_file
+        )
+        if setup_path.exists():
+            file_model_name = get_relative_model_name(setup_path, directory_path)
+            safe_file_name = get_safe_model_name(file_model_name)
+            file_ext = setup_path.suffix
+
+            components.append(
+                ImportFile(
+                    url=f"{base_url}/files/{safe_file_name}{file_ext}",
+                    name=file_model_name,
+                    alias="",
+                    type="sql" if file_ext == ".sql" else "trilogy",
+                    purpose="setup",
+                )
+            )
+
+    # Add all trilogy files (preql and sql) with purpose="source"
+    for trilogy_file in trilogy_files:
+        # Skip if already added as setup script
+        if any(
+            trilogy_file.samefile(s) if s.exists() else False for s in setup_scripts
+        ):
+            continue
+
+        file_model_name = get_relative_model_name(trilogy_file, directory_path)
+        safe_file_name = get_safe_model_name(file_model_name)
+        file_ext = trilogy_file.suffix
+
+        components.append(
+            ImportFile(
+                url=f"{base_url}/files/{safe_file_name}{file_ext}",
+                name=file_model_name,
+                alias="",
+                type="sql" if file_ext == ".sql" else "trilogy",
+                purpose="source",
+            )
+        )
+
+    # Add CSV files with purpose="data"
+    for csv_file in csv_files:
+        file_model_name = get_relative_model_name(csv_file, directory_path)
+        safe_file_name = get_safe_model_name(file_model_name)
+
+        components.append(
+            ImportFile(
+                url=f"{base_url}/files/{safe_file_name}.csv",
+                name=file_model_name,
+                alias=file_model_name,
+                type="csv",
+                purpose="data",
+            )
+        )
+
+    return ModelImport(
+        name=directory_path.name,
+        description=description,
+        engine=engine,
+        components=components,
+    )
+
+
+def find_file_content_by_name(file_name: str, directory_path: Path) -> str | None:
+
+    target_parts = Path(file_name.replace("-", "/")).parts
+
+    for model_file in find_all_model_files(directory_path):
+        relative_parts = model_file.relative_to(directory_path).parts
+        if relative_parts == target_parts:
+            return read_file_content(model_file)
+
+    return None

trilogy/scripts/serve_helpers/models.py

@@ -0,0 +1,38 @@
+"""Pydantic models for the serve command."""
+
+from pydantic import BaseModel, Field
+
+
+class ImportFile(BaseModel):
+    """Component file in a model import."""
+
+    url: str
+    name: str
+    alias: str = ""
+    purpose: str
+    type: str | None = None
+
+
+class ModelImport(BaseModel):
+    """Model import definition."""
+
+    name: str
+    engine: str
+    description: str
+    link: str = ""
+    tags: list[str] = Field(default_factory=list)
+    components: list[ImportFile]
+
+
+class StoreModelIndex(BaseModel):
+    """Individual model entry in the store index."""
+
+    name: str
+    url: str
+
+
+class StoreIndex(BaseModel):
+    """Store index containing list of available models."""
+
+    name: str
+    models: list[StoreModelIndex]

trilogy/scripts/single_execution.py

@@ -0,0 +1,131 @@
+import traceback
+from datetime import datetime
+from typing import Any, Union
+
+from trilogy import Executor
+from trilogy.core.statements.execute import PROCESSED_STATEMENT_TYPES
+from trilogy.scripts.display import (
+    FETCH_LIMIT,
+    ResultSet,
+    create_progress_context,
+    print_error,
+    print_info,
+    print_results_table,
+    show_statement_result,
+    show_statement_type,
+)
+
+
+def get_statement_type(statement: PROCESSED_STATEMENT_TYPES) -> str:
+    """Get the type/class name of a statement."""
+    return type(statement).__name__
+
+
+def execute_single_statement(
+    exec: Executor,
+    query: PROCESSED_STATEMENT_TYPES,
+    idx: int,
+    total_queries: int,
+    use_progress=False,
+) -> tuple[bool, ResultSet | None, Any, Union[Exception, None]]:
+    """Execute a single statement and handle results/errors consistently."""
+    # Log the statement type before execution
+    statement_type = get_statement_type(query)
+    if not use_progress:  # Only show type when not using progress bar
+        show_statement_type(idx, total_queries, statement_type)
+
+    start_time = datetime.now()
+
+    try:
+        raw_results = exec.execute_statement(query)
+        results = (
+            ResultSet(
+                rows=raw_results.fetchmany(FETCH_LIMIT + 1), columns=raw_results.keys()
+            )
+            if raw_results
+            else None
+        )
+        duration = datetime.now() - start_time
+
+        if not use_progress:
+            show_statement_result(idx, total_queries, duration, bool(results))
+
+        return True, results, duration, None
+
+    except Exception as e:
+        duration = datetime.now() - start_time
+
+        if not use_progress:
+            show_statement_result(idx, total_queries, duration, False, str(e), type(e))
+
+        return False, None, duration, e
+
+
+def execute_queries_with_progress(
+    exec: Executor, queries: list[PROCESSED_STATEMENT_TYPES]
+) -> Exception | None:
+    """Execute queries with Rich progress bar. Returns True if all succeeded, False if any failed."""
+    progress = create_progress_context()
+    results_to_print = []
+    exception = None
+
+    with progress:
+        task = progress.add_task("Executing statements...", total=len(queries))
+
+        for idx, query in enumerate(queries):
+            statement_type = get_statement_type(query)
+            progress.update(
+                task, description=f"Statement {idx+1}/{len(queries)} ({statement_type})"
+            )
+
+            success, results, duration, error = execute_single_statement(
+                exec, query, idx, len(queries), use_progress=True
+            )
+
+            if not success:
+                exception = error
+
+            # Store results for printing after progress is done
+            results_to_print.append(
+                (idx, len(queries), duration, success, results, error)
+            )
+            progress.advance(task)
+            if exception:
+                break
+
+    # Print all results after progress bar is finished
+    for idx, total_queries, duration, success, results, error in results_to_print:
+        if error:
+            show_statement_result(
+                idx, total_queries, duration, False, str(error), type(error)
+            )
+            print_error(f"Full traceback:\n{traceback.format_exc()}")
+        else:
+            show_statement_result(idx, total_queries, duration, bool(results))
+            if results and not error:
+                print_results_table(results)
+
+    return exception
+
+
+def execute_queries_simple(
+    exec: Executor, queries: list[PROCESSED_STATEMENT_TYPES]
+) -> Exception | None:
+    """Execute queries with simple output. Returns True if all succeeded, False if any failed."""
+    exception = None
+
+    for idx, query in enumerate(queries):
+        if len(queries) > 1:
+            print_info(f"Executing statement {idx+1} of {len(queries)}...")
+
+        success, results, duration, error = execute_single_statement(
+            exec, query, idx, len(queries), use_progress=False
+        )
+
+        if not success:
+            exception = error
+
+        if results and not error:
+            print_results_table(results)
+
+    return exception

trilogy/scripts/testing.py

@@ -0,0 +1,119 @@
+"""Testing commands (integration and unit) for Trilogy CLI."""
+
+from pathlib import Path as PathlibPath
+
+from click import UNPROCESSED, Path, argument, option, pass_context
+from click.exceptions import Exit
+
+from trilogy import Executor
+from trilogy.dialect.enums import Dialects
+from trilogy.scripts.common import (
+    CLIRuntimeParams,
+    handle_execution_exception,
+    validate_datasources,
+)
+from trilogy.scripts.dependency import ScriptNode
+from trilogy.scripts.parallel_execution import run_parallel_execution
+
+
+def execute_script_for_integration(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> None:
+    """Execute a script for the 'integration' command (parse + validate)."""
+    with open(node.path, "r") as f:
+        exec.parse_text(f.read())
+    validate_datasources(exec, mock=False, quiet=quiet)
+
+
+def execute_script_for_unit(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> None:
+    """Execute a script for the 'unit' command (parse + mock validate)."""
+    with open(node.path, "r") as f:
+        exec.parse_text(f.read())
+    validate_datasources(exec, mock=True, quiet=quiet)
+
+
+@argument("input", type=Path())
+@argument("dialect", type=str, required=False)
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@argument("conn_args", nargs=-1, type=UNPROCESSED)
+@pass_context
+def integration(
+    ctx, input, dialect: str | None, param, parallelism: int | None, config, conn_args
+):
+    """Run integration tests on Trilogy scripts."""
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects(dialect) if dialect else None,
+        parallelism=parallelism,
+        param=param,
+        conn_args=conn_args,
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_integration,
+            execution_mode="integration",
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)
+
+
+@argument("input", type=Path())
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@pass_context
+def unit(
+    ctx,
+    input,
+    param,
+    parallelism: int | None,
+    config,
+):
+    """Run unit tests on Trilogy scripts with mocked datasources."""
+    # Build CLI runtime params (unit tests always use DuckDB)
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects.DUCK_DB,
+        parallelism=parallelism,
+        param=param,
+        conn_args=(),
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_unit,
+            execution_mode="unit",
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)

trilogy/scripts/trilogy.py

@@ -0,0 +1,68 @@
+from click import group, option, pass_context
+
+from trilogy.scripts.agent import agent
+from trilogy.scripts.agent_info import agent_info
+from trilogy.scripts.display import set_rich_mode, show_debug_mode
+from trilogy.scripts.fmt import fmt
+from trilogy.scripts.ingest import ingest
+from trilogy.scripts.init import init
+from trilogy.scripts.plan import plan
+from trilogy.scripts.run import run
+from trilogy.scripts.serve import serve
+from trilogy.scripts.testing import integration, unit
+
+set_rich_mode = set_rich_mode
+
+# Default parallelism level
+DEFAULT_PARALLELISM = 2
+
+
+AGENT_NOTICE = """
+NOTE FOR AI AGENTS (Claude, GPT, etc.): For comprehensive usage documentation,
+run 'trilogy agent-info' to get a detailed AGENTS.md-style guide with all
+commands, options, and usage examples.
+"""
+
+
+@group(epilog=AGENT_NOTICE)
+@option("--debug", default=False, help="Enable debug mode")
+@pass_context
+def cli(ctx, debug: bool):
+    """Trilogy CLI - A beautiful data productivity tool."""
+    ctx.ensure_object(dict)
+    ctx.obj["DEBUG"] = debug
+
+    if debug:
+        show_debug_mode()
+
+
+# Add commands to the group
+cli.command("init")(init)
+cli.command("ingest", context_settings=dict(ignore_unknown_options=True))(ingest)
+cli.command("fmt", context_settings=dict(ignore_unknown_options=True))(fmt)
+cli.command(
+    "unit",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)(unit)
+cli.command(
+    "integration",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)(integration)
+cli.command(
+    "run",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)(run)
+cli.command("agent")(agent)
+cli.command("agent-info")(agent_info)
+cli.command("plan")(plan)
+cli.command("serve")(serve)
+
+
+if __name__ == "__main__":
+    cli()

trilogy/std/color.preql

@@ -0,0 +1,3 @@
+
+
+type hex string; # Hex color value, e.g. #FF5733

trilogy/std/date.preql

@@ -0,0 +1,13 @@
+
+
+type year int;
+type month int;
+type month_name string;
+type week int;
+type day int;
+type hour int;
+type minute int;
+type second int;
+type day_of_week int;
+type day_of_week_name string;
+

trilogy/std/display.preql

@@ -0,0 +1,18 @@
+
+
+type percent float; # Percentage value
+
+def calc_percent(a, b, digits=-1) -> 
+    case 
+        when digits =-1 then 
+            case 
+                when b = 0 then 0.0::numeric
+                else (a/b)::numeric
+            end
+        else round((
+            case 
+                when b = 0 then 0.0::float
+                else (a/b)::float
+            end
+            )::numeric, digits) 
+    end::numeric::percent;

trilogy/std/geography.preql

@@ -0,0 +1,22 @@
+
+
+## us types
+type us_state_short string; # US state abbreviation - ex MA, CA, NY
+type us_state string; # US state name - ex Massachusetts, California, New York
+
+type us_zip_code string; # US ZIP code
+
+## generic types
+type latitude float; # Latitude in degrees
+type longitude float; # Longitude in degrees
+type lat_long string; # Latitude and longitude in degrees, as a coordinate pair
+
+type city string; # City name
+type country string; # Full Country Name
+type country_code string; # ISO-3166 Country code - ex US, CA, GB
+type timezone string; # Timezone name
+type region string; # Region name
+
+
+## special formats
+type geojson string; # GeoJSON format for geographic data

trilogy/std/metric.preql

@@ -0,0 +1,15 @@
+# Length and distance units
+type m numeric;      # meters
+type km numeric;     # kilometers
+type cm numeric;     # centimeters
+type mm numeric;     # millimeters
+
+# Mass units
+type kg numeric;     # kilograms
+type g numeric;      # grams
+type tonne numeric;  # metric tons (1000 kg)
+
+# Force units
+type n numeric;      # newtons
+type kn numeric;     # kilonewtons (1000 N)
+type mn numeric;     # meganewtons (1,000,000 N)