qtype 0.0.16__py3-none-any.whl → 0.1.1__py3-none-any.whl

qtype/dsl/parser.py

@@ -0,0 +1,200 @@
+"""
+Parse YAML dictionaries into DSL models.
+
+This module handles the conversion of loaded YAML data into validated
+Pydantic DSL models, including custom type extraction and building.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import ValidationError
+
+from qtype.base.types import CustomTypeRegistry
+from qtype.dsl import model as dsl
+from qtype.dsl.custom_types import build_dynamic_types
+
+
+def _extract_type_definitions(data: dict[str, Any] | list) -> list[dict]:
+    """
+    Extract all custom type definitions from document.
+
+    Recursively finds type definitions in the document and any references.
+
+    Args:
+        data: Parsed YAML dictionary or list
+
+    Returns:
+        List of custom type definition dictionaries
+    """
+    types = []
+
+    # Handle list documents (e.g., ToolList, ModelList)
+    if isinstance(data, list):
+        return types
+
+    # Add types from Application documents
+    if isinstance(data, dict):
+        types.extend(data.get("types", []))
+
+    # Handle TypeList documents (root is a list of types)
+    if "root" in data:
+        root = data["root"]
+        if (
+            isinstance(root, list)
+            and len(root) > 0
+            and "properties" in root[0]
+        ):
+            types.extend(root)
+
+    # Recursively handle references
+    for ref in data.get("references", []):
+        types.extend(_extract_type_definitions(ref))
+
+    return types
+
+
+def _simplify_field_path(loc: tuple) -> str:
+    """
+    Simplify a Pydantic error location path for readability.
+
+    Removes verbose union type names and formats array indices.
+
+    Args:
+        loc: Error location tuple from Pydantic
+
+    Returns:
+        Simplified, readable field path string
+    """
+    simplified = []
+    for part in loc:
+        part_str = str(part)
+
+        # Skip union type discriminator paths (too verbose)
+        if "tagged-union" in part_str or "Union[" in part_str:
+            continue
+
+        # Skip Reference wrapper types
+        if part_str.startswith("Reference["):
+            continue
+
+        # Format numeric indices as array indices
+        if isinstance(part, int):
+            simplified.append(f"[{part}]")
+        else:
+            simplified.append(part_str)
+
+    return " -> ".join(simplified).replace(" -> [", "[")
+
+
+def _is_relevant_error(error: dict | Any) -> bool:
+    """
+    Determine if a validation error is relevant to show.
+
+    Filters out noise from union type matching attempts.
+
+    Args:
+        error: Pydantic error dictionary
+
+    Returns:
+        True if error should be shown to user
+    """
+    loc_str = " -> ".join(str(loc) for loc in error["loc"])
+    error_type = error["type"]
+
+    # Filter out "should be a valid list" errors for document types
+    # These are just union matching attempts
+    if error_type == "list_type" and any(
+        doc_type in loc_str
+        for doc_type in [
+            "AuthorizationProviderList",
+            "ModelList",
+            "ToolList",
+            "TypeList",
+            "VariableList",
+            "AgentList",
+            "FlowList",
+            "IndexList",
+        ]
+    ):
+        return False
+
+    # Filter out Reference wrapper errors about $ref field
+    # These are duplicates of actual validation errors
+    if "Reference[" in loc_str and "$ref" in error["loc"][-1]:
+        return False
+
+    return True
+
+
+def _format_validation_errors(
+    validation_error: ValidationError, source_name: str | None
+) -> str:
+    """
+    Format Pydantic validation errors in user-friendly way.
+
+    Args:
+        validation_error: ValidationError from Pydantic
+        source_name: Optional source file name for context
+
+    Returns:
+        Formatted error message string
+    """
+    # Filter and collect relevant errors
+    relevant_errors = [
+        error
+        for error in validation_error.errors()
+        if _is_relevant_error(error)
+    ]
+
+    if not relevant_errors:
+        # Fallback if all errors were filtered
+        error_msg = "Validation failed (see details above)"
+    else:
+        error_msg = "Validation failed:\n"
+        for error in relevant_errors[:5]:  # Show max 5 errors
+            loc_path = _simplify_field_path(error["loc"])
+            error_msg += f"  {loc_path}: {error['msg']}\n"
+
+        if len(relevant_errors) > 5:
+            error_msg += f"  ... and {len(relevant_errors) - 5} more errors\n"
+
+    if source_name:
+        error_msg = f"In {source_name}:\n{error_msg}"
+
+    return error_msg
+
+
+def parse_document(
+    yaml_data: dict[str, Any], source_name: str | None = None
+) -> tuple[dsl.DocumentType, CustomTypeRegistry]:
+    """
+    Parse validated YAML dictionary into DSL document.
+
+    Args:
+        yaml_data: Pre-loaded YAML dictionary
+        source_name: Optional source name for error messages
+
+    Returns:
+        Tuple of (DocumentType, CustomTypeRegistry)
+
+    Raises:
+        ValueError: If validation fails
+    """
+    # Extract and build custom types
+    type_defs = _extract_type_definitions(yaml_data)
+    custom_types = build_dynamic_types(type_defs)
+
+    # Validate with Pydantic
+    try:
+        document = dsl.Document.model_validate(
+            yaml_data, context={"custom_types": custom_types}
+        )
+    except ValidationError as e:
+        # Format validation errors nicely
+        error_msg = _format_validation_errors(e, source_name)
+        raise ValueError(error_msg) from e
+
+    # Extract root document from wrapper
+    return document.root, custom_types

qtype/dsl/types.py

@@ -0,0 +1,50 @@
+"""Utility types and functions for the DSL."""
+
+from __future__ import annotations
+
+from datetime import date, datetime, time
+
+from qtype.base.types import PrimitiveTypeEnum
+
+# Mapping of QType primitive types to Python types for internal representations
+PRIMITIVE_TO_PYTHON_TYPE = {
+    PrimitiveTypeEnum.audio: bytes,
+    PrimitiveTypeEnum.boolean: bool,
+    PrimitiveTypeEnum.bytes: bytes,
+    PrimitiveTypeEnum.date: date,
+    PrimitiveTypeEnum.datetime: datetime,
+    PrimitiveTypeEnum.int: int,
+    PrimitiveTypeEnum.file: bytes,  # Use bytes for file content
+    PrimitiveTypeEnum.float: float,
+    PrimitiveTypeEnum.image: bytes,  # Use bytes for image data
+    PrimitiveTypeEnum.text: str,
+    PrimitiveTypeEnum.time: time,  # Use time for time representation
+    PrimitiveTypeEnum.video: bytes,  # Use bytes for video data
+}
+
+PYTHON_TYPE_TO_PRIMITIVE_TYPE = {
+    bytes: PrimitiveTypeEnum.file,
+    bool: PrimitiveTypeEnum.boolean,
+    str: PrimitiveTypeEnum.text,
+    int: PrimitiveTypeEnum.int,
+    float: PrimitiveTypeEnum.float,
+    date: PrimitiveTypeEnum.date,
+    datetime: PrimitiveTypeEnum.datetime,
+    time: PrimitiveTypeEnum.time,
+    # TODO: decide on internal representation for images, video, and audio,
+    # or use annotation/hinting
+}
+
+
+def python_type_for_list(element_type: PrimitiveTypeEnum) -> type:
+    """
+    Get the Python list type for a given QType primitive element type.
+
+    Args:
+        element_type: The primitive type of the list elements
+
+    Returns:
+        The corresponding Python list type (e.g., list[str] for text elements)
+    """
+    element_python_type = PRIMITIVE_TO_PYTHON_TYPE[element_type]
+    return list[element_python_type]

qtype/interpreter/api.py

@@ -1,23 +1,23 @@
 from __future__ import annotations
 
-import logging
+import asyncio
+from contextlib import asynccontextmanager
 from pathlib import Path
 
-import pandas as pd
-from fastapi import FastAPI, HTTPException, Query
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
+from opentelemetry import trace
 
-from qtype.dsl.base_types import StepCardinality
-from qtype.interpreter.batch.flow import batch_execute_flow
-from qtype.interpreter.batch.types import BatchConfig, ErrorMode
-from qtype.interpreter.flow import execute_flow
-from qtype.interpreter.typing import (
-    create_input_type_model,
-    create_output_type_model,
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.base.secrets import create_secret_manager
+from qtype.interpreter.endpoints import (
+    create_rest_endpoint,
+    create_streaming_endpoint,
 )
-from qtype.semantic.model import Application, Flow
+from qtype.interpreter.metadata_api import create_metadata_endpoints
+from qtype.semantic.model import Application
 
 
 class APIExecutor:
@@ -51,7 +51,36 @@ class APIExecutor:
         if servers is not None:
             fast_api_args["servers"] = servers
 
-        app = FastAPI(title=name or "QType API", **fast_api_args)
+        # Create secret manager if configured
+        secret_manager = create_secret_manager(self.definition.secret_manager)
+
+        # Create lifespan context manager for telemetry
+        @asynccontextmanager
+        async def lifespan(app: FastAPI):
+            """Manage telemetry lifecycle during app startup/shutdown."""
+            tracer_provider = None
+            if self.definition.telemetry:
+                from qtype.interpreter.telemetry import register
+
+                tracer_provider = register(
+                    self.definition.telemetry,
+                    project_id=name or self.definition.id,
+                    secret_manager=secret_manager,
+                )
+            yield
+            # Fire off telemetry shutdown in background for fast reloads
+            if tracer_provider is not None:
+
+                async def shutdown_telemetry():
+                    tracer_provider.force_flush(timeout_millis=1000)
+                    tracer_provider.shutdown()
+
+                asyncio.create_task(shutdown_telemetry())
+
+        # Create FastAPI app with lifespan
+        app = FastAPI(
+            title=name or "QType API", lifespan=lifespan, **fast_api_args
+        )
 
         # Serve static UI files if they exist
         if ui_enabled:
@@ -71,132 +100,24 @@ class APIExecutor:
                     StaticFiles(directory=str(ui_dir), html=True),
                     name="ui",
                 )
-                app.get("/")(lambda: RedirectResponse(url="/ui"))
+                app.get("/", include_in_schema=False)(
+                    lambda: RedirectResponse(url="/ui")
+                )
 
-        flows = self.definition.flows if self.definition.flows else []
+        # Create metadata endpoints for flow discovery
+        create_metadata_endpoints(app, self.definition)
 
-        # Dynamically generate POST endpoints for each flow
-        for flow in flows:
-            if flow.mode == "Chat":
-                # For chat, we can create a single endpoint
-                # that handles the chat interactions
-                from qtype.interpreter.chat.chat_api import (
-                    create_chat_flow_endpoint,
-                )
+        # Create executor context
+        context = ExecutorContext(
+            secret_manager=secret_manager,
+            tracer=trace.get_tracer(__name__),
+        )
 
-                create_chat_flow_endpoint(app, flow)
-            else:
-                self._create_flow_endpoint(app, flow)
+        # Create unified invoke endpoints for each flow
+        flows = self.definition.flows if self.definition.flows else []
+        for flow in flows:
+            if flow.interface is not None:
+                create_streaming_endpoint(app, flow, context)
+            create_rest_endpoint(app, flow, context)
 
         return app
-
-    def _create_flow_endpoint(self, app: FastAPI, flow: Flow) -> None:
-        """Create a dynamic POST endpoint for a specific flow."""
-        flow_id = flow.id
-
-        # determine if this is a batch inference
-        is_batch = flow.cardinality == StepCardinality.many
-
-        # Create dynamic request and response models for this flow
-        RequestModel = create_input_type_model(flow, is_batch)
-        ResponseModel = create_output_type_model(flow, is_batch)
-
-        # Create the endpoint function with proper model binding
-        if is_batch:
-
-            def execute_flow_endpoint(  # type: ignore
-                request: RequestModel,  # type: ignore
-                error_mode: ErrorMode = Query(
-                    default=ErrorMode.FAIL,
-                    description="Error handling mode for batch processing",
-                ),
-            ) -> ResponseModel:  # type: ignore
-                try:
-                    # Make a copy of the flow to avoid modifying the original
-                    # TODO: Use session to ensure memory is not used across requests.
-                    flow_copy = flow.model_copy(deep=True)
-                    # convert the inputs into a dataframe with a single row
-                    inputs = pd.DataFrame(
-                        [i.model_dump() for i in request.inputs]  # type: ignore
-                    )
-
-                    # Execute the flow
-                    results, errors = batch_execute_flow(
-                        flow_copy,
-                        inputs,
-                        batch_config=BatchConfig(error_mode=error_mode),
-                    )
-
-                    response_data = {
-                        "flow_id": flow_id,
-                        "outputs": results.to_dict(orient="records"),
-                        "errors": errors.to_dict(orient="records"),
-                        "num_results": len(results),
-                        "num_errors": len(errors),
-                        "num_inputs": len(inputs),
-                        "status": "success" if len(errors) == 0 else "partial",
-                    }
-
-                    # Return the response using the dynamic model
-                    return ResponseModel(**response_data)  # type: ignore
-
-                except Exception as e:
-                    logging.error("Batch Flow Execution Failed", exc_info=e)
-                    raise HTTPException(
-                        status_code=500,
-                        detail=f"Batch flow execution failed: {str(e)}",
-                    )
-        else:
-
-            def execute_flow_endpoint(request: RequestModel) -> ResponseModel:  # type: ignore
-                try:
-                    # Make a copy of the flow to avoid modifying the original
-                    # TODO: Use session to ensure memory is not used across requests.
-                    flow_copy = flow.model_copy(deep=True)
-                    # Set input values on the flow variables
-                    if flow_copy.inputs:
-                        for var in flow_copy.inputs:
-                            # Get the value from the request using the variable ID
-                            request_dict = request.model_dump()  # type: ignore
-                            if var.id in request_dict:
-                                var.value = getattr(request, var.id)
-                            elif not var.is_set():
-                                raise HTTPException(
-                                    status_code=400,
-                                    detail=f"Required input '{var.id}' not provided",
-                                )
-                    # Execute the flow
-                    result_vars = execute_flow(flow_copy)
-
-                    # Extract output values
-                    outputs = {var.id: var.value for var in result_vars}
-
-                    response_data = {
-                        "flow_id": flow_id,
-                        "outputs": outputs,
-                        "status": "success",
-                    }
-
-                    # Return the response using the dynamic model
-                    return ResponseModel(**response_data)  # type: ignore
-
-                except Exception as e:
-                    raise HTTPException(
-                        status_code=500,
-                        detail=f"Flow execution failed: {str(e)}",
-                    )
-
-        # Set the function annotations properly for FastAPI
-        execute_flow_endpoint.__annotations__ = {
-            "request": RequestModel,
-            "error_mode": ErrorMode,
-            "return": ResponseModel,
-        }
-
-        # Add the endpoint with explicit models
-        app.post(
-            f"/flows/{flow_id}",
-            tags=["flow"],
-            description=flow.description or "Execute a flow",
-            response_model=ResponseModel,
-        )(execute_flow_endpoint)

qtype/interpreter/auth/aws.py

@@ -17,6 +17,7 @@ from botocore.exceptions import (  # type: ignore[import-untyped]
 )
 
 from qtype.interpreter.auth.cache import cache_auth, get_cached_auth
+from qtype.interpreter.base.secrets import SecretManagerBase
 from qtype.semantic.model import AWSAuthProvider
 
 
@@ -56,7 +57,10 @@ def _is_session_valid(session: boto3.Session) -> bool:
 
 
 @contextmanager
-def aws(aws_provider: AWSAuthProvider) -> Generator[boto3.Session, None, None]:
+def aws(
+    aws_provider: AWSAuthProvider,
+    secret_manager: SecretManagerBase,
+) -> Generator[boto3.Session, None, None]:
     """
     Create a boto3 Session using AWS authentication provider configuration.
 
@@ -113,7 +117,7 @@ def aws(aws_provider: AWSAuthProvider) -> Generator[boto3.Session, None, None]:
             return
 
         # Cache miss or invalid session - create new session
-        session = _create_session(aws_provider)
+        session = _create_session(aws_provider, secret_manager)
 
         # Validate the session by attempting to get credentials
         credentials = session.get_credentials()
@@ -137,12 +141,16 @@ def aws(aws_provider: AWSAuthProvider) -> Generator[boto3.Session, None, None]:
         ) from e
 
 
-def _create_session(aws_provider: AWSAuthProvider) -> boto3.Session:
+def _create_session(
+    aws_provider: AWSAuthProvider,
+    secret_manager: SecretManagerBase,
+) -> boto3.Session:
     """
     Create a boto3 Session based on the AWS provider configuration.
 
     Args:
         aws_provider: AWSAuthProvider with authentication details
+        secret_manager: Secret manager for resolving SecretReferences
 
     Returns:
         boto3.Session: Configured session
@@ -162,14 +170,16 @@ def _create_session(aws_provider: AWSAuthProvider) -> boto3.Session:
         session_kwargs["profile_name"] = aws_provider.profile_name
 
     elif aws_provider.access_key_id and aws_provider.secret_access_key:
-        # Use direct credentials
-        session_kwargs["aws_access_key_id"] = aws_provider.access_key_id
-        session_kwargs["aws_secret_access_key"] = (
-            aws_provider.secret_access_key
-        )
+        # Use direct credentials - resolve secrets
+        context = f"AWS auth provider '{aws_provider.id}'"
+        access_key = secret_manager(aws_provider.access_key_id, context)
+        secret_key = secret_manager(aws_provider.secret_access_key, context)
+        session_kwargs["aws_access_key_id"] = access_key
+        session_kwargs["aws_secret_access_key"] = secret_key
 
         if aws_provider.session_token:
-            session_kwargs["aws_session_token"] = aws_provider.session_token
+            session_token = secret_manager(aws_provider.session_token, context)
+            session_kwargs["aws_session_token"] = session_token
 
     # Create the base session
     session = boto3.Session(**session_kwargs)