pytrilogy 0.3.149__cp313-cp313-win_amd64.whl

trilogy/scripts/plan.py

@@ -0,0 +1,189 @@
+"""Plan command for Trilogy CLI - shows execution order without running."""
+
+import json
+from pathlib import Path as PathlibPath
+from typing import Any
+
+import networkx as nx
+from click import Path, argument, option, pass_context
+from click.exceptions import Exit
+
+from trilogy.scripts.common import (
+    handle_execution_exception,
+    resolve_input_information,
+)
+from trilogy.scripts.dependency import ETLDependencyStrategy, ScriptNode
+from trilogy.scripts.display import print_error, print_info, show_execution_plan
+from trilogy.scripts.parallel_execution import ParallelExecutor
+
+
+def safe_relative_path(path: PathlibPath, root: PathlibPath) -> str:
+    """Get path relative to root, falling back to absolute if not a subpath.
+
+    This handles the case where imports pull in files from outside the root
+    directory (e.g., `import ../shared/utils.preql`).
+    """
+    try:
+        return str(path.relative_to(root))
+    except ValueError:
+        return str(path)
+
+
+def graph_to_json(graph: nx.DiGraph, root: PathlibPath) -> dict[str, Any]:
+    """Convert dependency graph to JSON-serializable dict."""
+    nodes = [
+        {"id": safe_relative_path(node.path, root), "path": str(node.path)}
+        for node in graph.nodes()
+    ]
+    edges = [
+        {
+            "from": safe_relative_path(from_node.path, root),
+            "to": safe_relative_path(to_node.path, root),
+        }
+        for from_node, to_node in graph.edges()
+    ]
+    return {"nodes": nodes, "edges": edges}
+
+
+def get_execution_levels(graph: nx.DiGraph) -> list[list[ScriptNode]]:
+    """Get execution levels (BFS layers) from dependency graph."""
+    if not graph.nodes():
+        return []
+
+    levels: list[list[ScriptNode]] = []
+    remaining_deps = {node: graph.in_degree(node) for node in graph.nodes()}
+    completed: set[ScriptNode] = set()
+
+    while len(completed) < len(graph.nodes()):
+        ready = [
+            node
+            for node in graph.nodes()
+            if remaining_deps[node] == 0 and node not in completed
+        ]
+        if not ready:
+            break
+        levels.append(ready)
+        for node in ready:
+            completed.add(node)
+            for dependent in graph.successors(node):
+                remaining_deps[dependent] -= 1
+
+    return levels
+
+
+def format_plan_text(
+    graph: nx.DiGraph, root: PathlibPath
+) -> tuple[list[str], list[tuple[str, str]], list[list[str]]]:
+    """Format plan for text display."""
+    nodes = [safe_relative_path(node.path, root) for node in graph.nodes()]
+    edges = [
+        (
+            safe_relative_path(from_node.path, root),
+            safe_relative_path(to_node.path, root),
+        )
+        for from_node, to_node in graph.edges()
+    ]
+    levels = get_execution_levels(graph)
+    execution_order = [
+        [safe_relative_path(node.path, root) for node in level] for level in levels
+    ]
+    return nodes, edges, execution_order
+
+
+@argument("input", type=Path())
+@option(
+    "--output",
+    "-o",
+    type=Path(),
+    default=None,
+    help="Output file path for the plan",
+)
+@option(
+    "--json",
+    "json_format",
+    is_flag=True,
+    default=False,
+    help="Output plan as JSON graph with nodes and edges",
+)
+@option(
+    "--config",
+    type=Path(exists=True),
+    help="Path to trilogy.toml configuration file",
+)
+@pass_context
+def plan(ctx, input: str, output: str | None, json_format: bool, config: str | None):
+    """Show execution plan without running scripts."""
+    try:
+        pathlib_input = PathlibPath(input)
+        config_path = PathlibPath(config) if config else None
+
+        # Resolve input to validate it exists
+        _ = list(resolve_input_information(input, config_path)[0])
+
+        if not pathlib_input.exists():
+            print_error(f"Input path '{input}' does not exist.")
+            raise Exit(1)
+
+        # Set up parallel executor with ETL strategy
+        parallel_exec = ParallelExecutor(
+            max_workers=1,
+            dependency_strategy=ETLDependencyStrategy(),
+        )
+
+        # Get execution plan
+        if pathlib_input.is_dir():
+            root = pathlib_input
+            graph = parallel_exec.get_folder_execution_plan(pathlib_input)
+        elif pathlib_input.is_file():
+            root = pathlib_input.parent
+            graph = parallel_exec.get_execution_plan([pathlib_input])
+        else:
+            print_error(f"Input path '{input}' is not a file or directory.")
+            raise Exit(1)
+
+        if json_format:
+            plan_data = graph_to_json(graph, root)
+            levels = get_execution_levels(graph)
+            plan_data["execution_order"] = [
+                [safe_relative_path(node.path, root) for node in level]
+                for level in levels
+            ]
+            json_output = json.dumps(plan_data, indent=2)
+
+            if output:
+                output_path = PathlibPath(output)
+                output_path.write_text(json_output)
+                print_info(f"Plan written to {output_path}")
+            else:
+                print(json_output)
+        else:
+            nodes, edges, execution_order = format_plan_text(graph, root)
+
+            if output:
+                output_path = PathlibPath(output)
+                lines = []
+                lines.append("Execution Plan")
+                lines.append(f"Scripts: {len(nodes)}")
+                lines.append(f"Dependencies: {len(edges)}")
+                lines.append(f"Execution Levels: {len(execution_order)}")
+                lines.append("")
+                if execution_order:
+                    lines.append("Execution Order:")
+                    for level_idx, level_scripts in enumerate(execution_order):
+                        lines.append(
+                            f"  Level {level_idx + 1}: {', '.join(level_scripts)}"
+                        )
+                    lines.append("")
+                if edges:
+                    lines.append("Dependencies:")
+                    for from_node, to_node in edges:
+                        lines.append(f"  {from_node} -> {to_node}")
+                output_path.write_text("\n".join(lines))
+                print_info(f"Plan written to {output_path}")
+            else:
+                show_execution_plan(nodes, edges, execution_order)
+
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=ctx.obj.get("DEBUG", False))

trilogy/scripts/refresh.py

@@ -0,0 +1,161 @@
+"""Refresh command for Trilogy CLI - refreshes stale assets."""
+
+from pathlib import Path as PathlibPath
+
+from click import UNPROCESSED, Path, argument, option, pass_context
+from click.exceptions import Exit
+
+from trilogy import Executor
+from trilogy.core.statements.execute import ProcessedValidateStatement
+from trilogy.dialect.enums import Dialects
+from trilogy.execution.state.state_store import (
+    DatasourceWatermark,
+    refresh_stale_assets,
+)
+from trilogy.scripts.common import (
+    CLIRuntimeParams,
+    ExecutionStats,
+    count_statement_stats,
+    handle_execution_exception,
+)
+from trilogy.scripts.dependency import ScriptNode
+from trilogy.scripts.parallel_execution import ExecutionMode, run_parallel_execution
+
+# Module-level flag for printing watermarks (set by CLI)
+_print_watermarks = False
+
+
+def _format_watermarks(watermarks: dict[str, DatasourceWatermark]) -> None:
+    """Print watermark information for all datasources."""
+    from trilogy.scripts.display import print_info
+
+    print_info("Watermarks:")
+    for ds_id, watermark in sorted(watermarks.items()):
+        if not watermark.keys:
+            print_info(f"  {ds_id}: (no watermarks)")
+            continue
+        for key_name, update_key in watermark.keys.items():
+            print_info(
+                f"  {ds_id}.{key_name}: {update_key.value} ({update_key.type.value})"
+            )
+
+
+def execute_script_for_refresh(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> ExecutionStats:
+    """Execute a script for the 'refresh' command - parse and refresh stale assets."""
+    from trilogy.scripts.display import print_info, print_success, print_warning
+
+    validation = []
+    with open(node.path, "r") as f:
+        statements = exec.parse_text(f.read())
+
+    for x in statements:
+        if isinstance(x, ProcessedValidateStatement):
+            validation.append(x)
+
+    stats = count_statement_stats([])
+
+    def on_stale_found(stale_count: int, root_assets: int, all_assets: int) -> None:
+        if stale_count == 0 and not quiet:
+            print_info(
+                f"No stale assets found in {node.path.name} ({root_assets}/{all_assets} root assets)"
+            )
+        elif not quiet:
+            print_warning(f"Found {stale_count} stale asset(s) in {node.path.name}")
+
+    def on_refresh(asset_id: str, reason: str) -> None:
+        if not quiet:
+            print_info(f"  Refreshing {asset_id}: {reason}")
+
+    def on_watermarks(watermarks: dict[str, DatasourceWatermark]) -> None:
+        if _print_watermarks:
+            _format_watermarks(watermarks)
+
+    result = refresh_stale_assets(
+        exec,
+        on_stale_found=on_stale_found,
+        on_refresh=on_refresh,
+        on_watermarks=on_watermarks,
+    )
+    stats.update_count = result.refreshed_count
+
+    for x in validation:
+        exec.execute_statement(x)
+        stats = count_statement_stats([x])
+
+    if result.had_stale and not quiet:
+        print_success(
+            f"Refreshed {result.refreshed_count} asset(s) in {node.path.name}"
+        )
+
+    return stats
+
+
+@argument("input", type=Path())
+@argument("dialect", type=str, required=False)
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@option(
+    "--print-watermarks",
+    is_flag=True,
+    default=False,
+    help="Print watermark values for all datasources before refreshing",
+)
+@option(
+    "--env",
+    "-e",
+    multiple=True,
+    help="Set environment variables as KEY=VALUE pairs",
+)
+@argument("conn_args", nargs=-1, type=UNPROCESSED)
+@pass_context
+def refresh(
+    ctx,
+    input,
+    dialect: str | None,
+    param,
+    parallelism: int | None,
+    config,
+    print_watermarks,
+    env,
+    conn_args,
+):
+    """Refresh stale assets in Trilogy scripts.
+
+    Parses each script, identifies datasources marked as 'root' (source of truth),
+    compares watermarks to find stale derived assets, and refreshes them.
+    """
+    global _print_watermarks
+    _print_watermarks = print_watermarks
+
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects(dialect) if dialect else None,
+        parallelism=parallelism,
+        param=param,
+        conn_args=conn_args,
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+        env=env,
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_refresh,
+            execution_mode=ExecutionMode.REFRESH,
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)

trilogy/scripts/run.py

@@ -0,0 +1,79 @@
+"""Run command for Trilogy CLI."""
+
+from pathlib import Path as PathlibPath
+
+from click import UNPROCESSED, Path, argument, option, pass_context
+from click.exceptions import Exit
+
+from trilogy import Executor
+from trilogy.dialect.enums import Dialects
+from trilogy.scripts.common import (
+    CLIRuntimeParams,
+    ExecutionStats,
+    execute_script_with_stats,
+    handle_execution_exception,
+)
+from trilogy.scripts.dependency import ScriptNode
+from trilogy.scripts.parallel_execution import ExecutionMode, run_parallel_execution
+
+
+def execute_script_for_run(
+    exec: Executor, node: ScriptNode, quiet: bool = False
+) -> ExecutionStats:
+    """Execute a script for the 'run' command (parallel execution mode)."""
+    return execute_script_with_stats(exec, node.path, run_statements=True)
+
+
+@argument("input", type=Path())
+@argument("dialect", type=str, required=False)
+@option("--param", multiple=True, help="Environment parameters as key=value pairs")
+@option(
+    "--parallelism",
+    "-p",
+    default=None,
+    help="Maximum parallel workers for directory execution",
+)
+@option(
+    "--config", type=Path(exists=True), help="Path to trilogy.toml configuration file"
+)
+@option(
+    "--env",
+    "-e",
+    multiple=True,
+    help="Set environment variables as KEY=VALUE pairs",
+)
+@argument("conn_args", nargs=-1, type=UNPROCESSED)
+@pass_context
+def run(
+    ctx,
+    input,
+    dialect: str | None,
+    param,
+    parallelism: int | None,
+    config,
+    env,
+    conn_args,
+):
+    """Execute a Trilogy script or query."""
+    cli_params = CLIRuntimeParams(
+        input=input,
+        dialect=Dialects(dialect) if dialect else None,
+        parallelism=parallelism,
+        param=param,
+        conn_args=conn_args,
+        debug=ctx.obj["DEBUG"],
+        config_path=PathlibPath(config) if config else None,
+        execution_strategy="eager_bfs",
+        env=env,
+    )
+
+    try:
+        run_parallel_execution(
+            cli_params=cli_params,
+            execution_fn=execute_script_for_run,
+            execution_mode=ExecutionMode.RUN,
+        )
+    except Exit:
+        raise
+    except Exception as e:
+        handle_execution_exception(e, debug=cli_params.debug)

trilogy/scripts/serve.py

@@ -0,0 +1,202 @@
+"""Serve command for Trilogy CLI."""
+
+import sys
+from pathlib import Path as PathlibPath
+from urllib.parse import quote
+
+from click import Path, argument, option, pass_context
+
+from trilogy.execution.config import load_config_file
+from trilogy.scripts.common import find_trilogy_config
+from trilogy.scripts.serve_helpers import (
+    find_trilogy_files,
+    get_relative_model_name,
+    get_safe_model_name,
+)
+
+
+def check_fastapi_available() -> bool:
+    """Check if FastAPI and uvicorn are available."""
+    try:
+        import fastapi  # noqa: F401
+        import uvicorn  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def create_app(app, engine: str, directory_path: PathlibPath, host: str, port: int):
+    from fastapi import HTTPException
+    from fastapi.middleware.cors import CORSMiddleware
+    from fastapi.responses import PlainTextResponse
+
+    from trilogy.scripts.serve_helpers import (
+        ModelImport,
+        StoreIndex,
+        find_all_model_files,
+        find_file_content_by_name,
+        find_model_by_name,
+        generate_model_index,
+    )
+
+    url_host = "localhost" if host == "0.0.0.0" else host
+    if port in (80, 443):
+        base_url = f"http://{url_host}"
+    else:
+        base_url = f"http://{url_host}:{port}"
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @app.get("/")
+    async def root():
+        """Root endpoint with server information."""
+        file_count = len(find_all_model_files(directory_path))
+        return {
+            "message": "Trilogy Model Server",
+            "description": f"Serving model '{directory_path.name}' with {file_count} files from {directory_path}",
+            "endpoints": {
+                "/index.json": "Get list of available models",
+                "/models/<model-name>.json": "Get specific model details",
+            },
+        }
+
+    @app.get("/index.json", response_model=StoreIndex)
+    async def get_index() -> StoreIndex:
+        """Return the store index with list of available models."""
+        return StoreIndex(
+            name=f"Trilogy Models - {directory_path.name}",
+            models=generate_model_index(directory_path, base_url, engine),
+        )
+
+    @app.get("/models/{model_name}.json", response_model=ModelImport)
+    async def get_model(model_name: str) -> ModelImport:
+        """Return a specific model by name."""
+        model = find_model_by_name(model_name, directory_path, base_url, engine)
+        if model is None:
+            raise HTTPException(status_code=404, detail="Model not found")
+        return model
+
+    @app.get("/files/{file_name}")
+    async def get_file(file_name: str):
+        """Return the raw .preql or .sql file content."""
+        content = find_file_content_by_name(file_name, directory_path)
+        if content is None:
+            raise HTTPException(status_code=404, detail="File not found")
+        return PlainTextResponse(content=content)
+
+    print(f"Starting Trilogy Model Server on http://{host}:{port}")
+    print(f"Serving model '{directory_path.name}' from: {directory_path}")
+    print(f"Engine: {engine}")
+    print(f"Access the index at: http://{host}:{port}/index.json")
+    print(
+        f"Found {len(find_all_model_files(directory_path))} model files (.preql, .sql, .csv)"
+    )
+    return app
+
+
+@argument("path", type=Path(exists=True, file_okay=True, dir_okay=True))
+@argument("engine", type=str, required=False, default="generic")
+@option("--port", "-p", default=8100, help="Port to run the server on")
+@option("--host", "-h", default="0.0.0.0", help="Host to bind the server to")
+@option("--timeout", "-t", default=None, type=float, help="Shutdown after N seconds")
+@pass_context
+def serve(ctx, path: str, engine: str, port: int, host: str, timeout: float | None):
+    """Start a FastAPI server to expose Trilogy models from a directory or file."""
+    if not check_fastapi_available():
+        print(
+            "Error: FastAPI and uvicorn are required for the serve command.\n"
+            "Please install with: pip install pytrilogy[serve]",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    import uvicorn
+    from fastapi import FastAPI
+
+    from trilogy import __version__
+
+    path_obj = PathlibPath(path).resolve()
+
+    # Determine directory and target file
+    if path_obj.is_file():
+        directory_path = path_obj.parent
+        target_file = path_obj
+    else:
+        directory_path = path_obj
+        target_file = None
+
+    # Load trilogy.toml to get engine dialect if not explicitly provided
+    config_path = find_trilogy_config(directory_path)
+    if config_path and engine == "generic":
+        try:
+            runtime_config = load_config_file(config_path)
+            if runtime_config.engine_dialect:
+                engine = runtime_config.engine_dialect.value
+        except Exception:
+            pass
+
+    # Use localhost instead of 0.0.0.0 in URLs so they resolve properly
+    app = FastAPI(title="Trilogy Model Server", version=__version__)
+
+    create_app(app, engine, directory_path, host, port)
+
+    # Generate Trilogy Studio URL
+    url_host = "localhost" if host == "0.0.0.0" else host
+    base_url = (
+        f"http://{url_host}:{port}" if port not in (80, 443) else f"http://{url_host}"
+    )
+
+    # Find target file if not specified
+    if target_file is None:
+        trilogy_files = find_trilogy_files(directory_path)
+        if trilogy_files:
+            target_file = trilogy_files[0]
+
+    if target_file:
+        # The model URL uses the directory name (not the file name)
+        model_safe_name = get_safe_model_name(directory_path.name)
+        model_url = f"{base_url}/models/{model_safe_name}.json"
+        store_url = base_url
+
+        # The asset name is the specific file within the model
+        asset_name = get_relative_model_name(target_file, directory_path)
+
+        # URL-encode the parameters
+        studio_url = (
+            f"https://trilogydata.dev/trilogy-studio-core/#"
+            f"import={quote(model_url)}&"
+            f"assetType=trilogy&"
+            f"assetName={quote(asset_name)}&"
+            f"modelName={quote(directory_path.name)}&"
+            f"connection={quote(engine)}&"
+            f"store={quote(store_url)}"
+        )
+
+        print("\n" + "=" * 80)
+        print("Trilogy Studio Link:")
+        print(studio_url)
+        print("=" * 80 + "\n")
+
+    if timeout is not None:
+        import threading
+
+        config = uvicorn.Config(app, host=host, port=port)
+        server = uvicorn.Server(config)
+
+        def shutdown_after_timeout():
+            import time
+
+            time.sleep(timeout)
+            server.should_exit = True
+
+        timer = threading.Thread(target=shutdown_after_timeout, daemon=True)
+        timer.start()
+        server.run()
+    else:
+        uvicorn.run(app, host=host, port=port)

trilogy/scripts/serve_helpers/__init__.py

@@ -0,0 +1,41 @@
+"""Helpers for the serve command."""
+
+from trilogy.scripts.serve_helpers.file_discovery import (
+    extract_description_from_file,
+    find_all_model_files,
+    find_csv_files,
+    find_preql_files,
+    find_sql_files,
+    find_trilogy_files,
+    get_relative_model_name,
+    get_safe_model_name,
+)
+from trilogy.scripts.serve_helpers.index_generation import (
+    find_file_content_by_name,
+    find_model_by_name,
+    generate_model_index,
+)
+from trilogy.scripts.serve_helpers.models import (
+    ImportFile,
+    ModelImport,
+    StoreIndex,
+    StoreModelIndex,
+)
+
+__all__ = [
+    "ImportFile",
+    "ModelImport",
+    "StoreIndex",
+    "StoreModelIndex",
+    "find_preql_files",
+    "find_sql_files",
+    "find_csv_files",
+    "find_trilogy_files",
+    "find_all_model_files",
+    "extract_description_from_file",
+    "get_relative_model_name",
+    "get_safe_model_name",
+    "generate_model_index",
+    "find_model_by_name",
+    "find_file_content_by_name",
+]