qtype 0.0.16__py3-none-any.whl → 0.1.1__py3-none-any.whl

qtype/application/commons/tools.py

@@ -180,7 +180,7 @@ def parse_duration_string(duration: str) -> int:
 
 def format_datetime(timestamp: datetime, format_string: str) -> str:
     """
-    Format a timestamp using a custom format string.
+    Format a timestamp using a custom format string that can be passed to strftime.
 
     Args:
         timestamp: Datetime object to format.

qtype/application/converters/tools_from_api.py

@@ -17,7 +17,7 @@ from openapi_parser.specification import (
     Security,
 )
 
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 from qtype.dsl.model import (
     APIKeyAuthProvider,
     APITool,
@@ -344,9 +344,9 @@ def to_api_tool(
         endpoint=endpoint,
         method=operation.method.value.upper(),
         auth=auth.id if auth else None,  # Use auth ID string instead of object
-        inputs=inputs if inputs else None,
-        outputs=outputs if outputs else None,
-        parameters=parameters if parameters else None,
+        inputs=inputs,
+        outputs=outputs,
+        parameters=parameters,
     )
 
 
@@ -394,7 +394,7 @@ def to_authorization_provider(
                     }
                 )
                 if any(flow.scopes for flow in security.flows.values())
-                else None,
+                else [],
             )
         case _:
             raise ValueError(

qtype/application/converters/tools_from_module.py

@@ -5,7 +5,7 @@ from typing import Any, Type, Union, get_args, get_origin
 from pydantic import BaseModel
 
 from qtype.application.converters.types import PYTHON_TYPE_TO_PRIMITIVE_TYPE
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 from qtype.dsl.model import (
     CustomType,
     ListType,
@@ -159,7 +159,7 @@ def _create_tool_from_function(
         module_path=func_info["module"],
         function_name=func_name,
         description=description,
-        inputs=inputs if inputs else None,
+        inputs=inputs,
         outputs=outputs,
     )
 

qtype/application/converters/types.py

@@ -1,47 +1,18 @@
-from datetime import date, datetime, time
-
-from qtype.dsl.base_types import PrimitiveTypeEnum
-
-"""
-Mapping of QType primitive types to Python types for internal representations.
 """
-PRIMITIVE_TO_PYTHON_TYPE = {
-    PrimitiveTypeEnum.audio: bytes,
-    PrimitiveTypeEnum.boolean: bool,
-    PrimitiveTypeEnum.bytes: bytes,
-    PrimitiveTypeEnum.date: date,
-    PrimitiveTypeEnum.datetime: datetime,
-    PrimitiveTypeEnum.int: int,
-    PrimitiveTypeEnum.file: bytes,  # Use bytes for file content
-    PrimitiveTypeEnum.float: float,
-    PrimitiveTypeEnum.image: bytes,  # Use bytes for image data
-    PrimitiveTypeEnum.text: str,
-    PrimitiveTypeEnum.time: time,  # Use time for time representation
-    PrimitiveTypeEnum.video: bytes,  # Use bytes for video data
-}
+Re-export type mappings from DSL layer for backward compatibility.
 
-PYTHON_TYPE_TO_PRIMITIVE_TYPE = {
-    bytes: PrimitiveTypeEnum.file,
-    bool: PrimitiveTypeEnum.boolean,
-    str: PrimitiveTypeEnum.text,
-    int: PrimitiveTypeEnum.int,
-    float: PrimitiveTypeEnum.float,
-    date: PrimitiveTypeEnum.date,
-    datetime: PrimitiveTypeEnum.datetime,
-    time: PrimitiveTypeEnum.time,
-    # TODO: decide on internal representation for images, video, and audio, or use annotation/hinting
-}
-
-
-def python_type_for_list(element_type: PrimitiveTypeEnum) -> type:
-    """
-    Get the Python list type for a given QType primitive element type.
+This module maintains the application layer's interface while delegating
+to the DSL layer where these mappings are now defined.
+"""
 
-    Args:
-        element_type: The primitive type of the list elements
+from qtype.dsl.types import (
+    PRIMITIVE_TO_PYTHON_TYPE,
+    PYTHON_TYPE_TO_PRIMITIVE_TYPE,
+    python_type_for_list,
+)
 
-    Returns:
-        The corresponding Python list type (e.g., list[str] for text elements)
-    """
-    element_python_type = PRIMITIVE_TO_PYTHON_TYPE[element_type]
-    return list[element_python_type]
+__all__ = [
+    "PRIMITIVE_TO_PYTHON_TYPE",
+    "PYTHON_TYPE_TO_PRIMITIVE_TYPE",
+    "python_type_for_list",
+]

qtype/application/documentation.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any, Type, Union, get_args, get_origin
 
 import qtype.dsl.model as dsl
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 
 
 def _format_type_name(field_type: Any) -> str:

qtype/application/facade.py

@@ -2,74 +2,90 @@
 
 from __future__ import annotations
 
+import logging
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-from pydantic import BaseModel
-
-from qtype.base.logging import get_logger
-from qtype.base.types import CustomTypeRegistry, DocumentRootType, PathLike
-from qtype.dsl.base_types import StepCardinality
-from qtype.dsl.model import Application as DSLApplication
-from qtype.dsl.model import DocumentType
-from qtype.interpreter.batch.types import BatchConfig
+from qtype.base.types import PathLike
 from qtype.semantic.model import Application as SemanticApplication
-from qtype.semantic.model import Variable
+from qtype.semantic.model import DocumentType as SemanticDocumentType
+
+# Note: There should be _zero_ imports here at the top that import qtype.interpreter.
+# That's the whole point of this facade - to avoid importing optional
+# dependencies unless these methods are called.
 
-logger = get_logger("application.facade")
+logger = logging.getLogger(__name__)
 
 
 class QTypeFacade:
     """
-    Simplified interface for all qtype operations.
+    Simplified interface for qtype operations.
 
-    This facade hides the complexity of coordinating between DSL, semantic,
-    and interpreter layers, providing a clean API for common operations.
+    This facade provides lazy-loading wrappers for operations that require
+    optional dependencies (interpreter package), allowing base qtype to work
+    without those dependencies installed.
     """
 
-    def load_dsl_document(
-        self, path: PathLike
-    ) -> tuple[DocumentRootType, CustomTypeRegistry]:
-        from qtype.loader import load_document
-
-        return load_document(Path(path).read_text(encoding="utf-8"))
-
-    def telemetry(self, spec: SemanticApplication) -> None:
-        if spec.telemetry:
+    def telemetry(self, spec: SemanticDocumentType) -> None:
+        if isinstance(spec, SemanticApplication) and spec.telemetry:
             logger.info(
                 f"Telemetry enabled with endpoint: {spec.telemetry.endpoint}"
             )
             # Register telemetry if needed
             from qtype.interpreter.telemetry import register
 
-            register(spec.telemetry, spec.id)
+            register(spec.telemetry, self.secret_manager(spec), spec.id)
 
-    def load_semantic_model(
-        self, path: PathLike
-    ) -> tuple[SemanticApplication, CustomTypeRegistry]:
-        """Load a document and return the resolved semantic model."""
-        from qtype.loader import load
+    def secret_manager(self, spec: SemanticDocumentType):
+        """
+        Create a secret manager based on the specification.
 
-        content = Path(path).read_text(encoding="utf-8")
-        return load(content)
+        Args:
+            spec: SemanticDocumentType specification
 
-    def execute_workflow(
+        Returns:
+            Secret manager instance
+        """
+        from qtype.interpreter.base.secrets import create_secret_manager
+
+        if isinstance(spec, SemanticApplication):
+            return create_secret_manager(spec.secret_manager)
+        else:
+            raise ValueError(
+                "Can't create secret manager for non-Application spec"
+            )
+
+    async def execute_workflow(
         self,
         path: PathLike,
-        inputs: dict | pd.DataFrame,
+        inputs: dict | Any,
         flow_name: str | None = None,
-        batch_config: BatchConfig | None = None,
         **kwargs: Any,
-    ) -> pd.DataFrame | list[Variable]:
-        """Execute a complete workflow from document to results."""
+    ) -> Any:
+        """
+        Execute a complete workflow from document to results.
+
+        Args:
+            path: Path to the QType specification file
+            inputs: Dictionary of input values or DataFrame for batch
+            flow_name: Optional name of flow to execute
+            **kwargs: Additional dependencies for execution
+
+        Returns:
+            DataFrame with results (one row per input)
+        """
+        import pandas as pd
+
+        from qtype.semantic.loader import load
+
         logger.info(f"Executing workflow from {path}")
 
         # Load the semantic application
-        semantic_model, type_registry = self.load_semantic_model(path)
+        semantic_model, type_registry = load(Path(path))
+        assert isinstance(semantic_model, SemanticApplication)
         self.telemetry(semantic_model)
 
-        # Find the flow to execute (inlined from _find_flow)
+        # Find the flow to execute
         if flow_name:
             target_flow = None
             for flow in semantic_model.flows:
@@ -83,49 +99,54 @@ class QTypeFacade:
                 target_flow = semantic_model.flows[0]
             else:
                 raise ValueError("No flows found in application")
-        if target_flow.cardinality == StepCardinality.many:
-            if isinstance(inputs, dict):
-                inputs = pd.DataFrame([inputs])
-            if not isinstance(inputs, pd.DataFrame):
-                raise ValueError(
-                    "Input must be a DataFrame for flows with 'many' cardinality"
-                )
-            from qtype.interpreter.batch.flow import batch_execute_flow
-
-            batch_config = batch_config or BatchConfig()
-            results, errors = batch_execute_flow(
-                target_flow, inputs, batch_config, **kwargs
-            )  # type: ignore
-            return results
+
+        # Convert inputs to DataFrame (normalize single dict to 1-row DataFrame)
+        if isinstance(inputs, dict):
+            input_df = pd.DataFrame([inputs])
+        elif isinstance(inputs, pd.DataFrame):
+            input_df = inputs
         else:
-            from qtype.interpreter.flow import execute_flow
+            raise ValueError(
+                f"Inputs must be dict or DataFrame, got {type(inputs)}"
+            )
 
-            for var in target_flow.inputs:
-                if var.id in inputs:
-                    var.value = inputs[var.id]
-            args = {**kwargs, **inputs}
-            return execute_flow(target_flow, **args)
+        # Create session
+        from qtype.interpreter.converters import (
+            dataframe_to_flow_messages,
+            flow_messages_to_dataframe,
+        )
+        from qtype.interpreter.types import Session
 
-    def visualize_application(self, path: PathLike) -> str:
-        """Visualize an application as Mermaid diagram."""
-        from qtype.semantic.visualize import visualize_application
+        session = Session(
+            session_id=kwargs.pop("session_id", "default"),
+            conversation_history=kwargs.pop("conversation_history", []),
+        )
 
-        semantic_model, _ = self.load_semantic_model(path)
-        return visualize_application(semantic_model)
+        # Convert DataFrame to FlowMessages
+        initial_messages = dataframe_to_flow_messages(input_df, session)
 
-    def convert_document(self, document: DocumentType) -> str:
-        """Convert a document to YAML format."""
-        # Wrap DSLApplication in Document if needed
-        wrapped_document: BaseModel = document
-        if isinstance(document, DSLApplication):
-            from qtype.dsl.model import Document
+        # Execute the flow
+        from opentelemetry import trace
 
-            wrapped_document = Document(root=document)
-        from pydantic_yaml import to_yaml_str
+        from qtype.interpreter.base.executor_context import ExecutorContext
+        from qtype.interpreter.flow import run_flow
 
-        return to_yaml_str(
-            wrapped_document, exclude_unset=True, exclude_none=True
+        secret_manager = self.secret_manager(semantic_model)
+        context = ExecutorContext(
+            secret_manager=secret_manager,
+            tracer=trace.get_tracer(__name__),
         )
+        results = await run_flow(
+            target_flow,
+            initial_messages,
+            context=context,
+            **kwargs,
+        )
+
+        # Convert results back to DataFrame
+        results_df = flow_messages_to_dataframe(results, target_flow)
+
+        return results_df
 
     def generate_aws_bedrock_models(self) -> list[dict[str, Any]]:
         """

qtype/base/types.py

@@ -3,11 +3,24 @@
 from __future__ import annotations
 
 import pathlib
-from typing import Any, Type, Union
+import types
+import typing
+from enum import Enum
+from typing import (
+    Any,
+    Generic,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
 
 from pydantic import BaseModel
-
-from qtype.dsl import model as dsl
+from pydantic import ConfigDict as PydanticConfigDict
+from pydantic import Field, model_validator
 
 # JSON-serializable value types
 JSONValue = Union[
@@ -20,10 +33,217 @@ JSONValue = Union[
     list["JSONValue"],
 ]
 
-# Configuration dictionary type
-ConfigDict = dict[str, Any]
-
 # Path-like type (string or Path object)
 PathLike = Union[str, pathlib.Path]
+
 CustomTypeRegistry = dict[str, Type[BaseModel]]
-DocumentRootType = dsl.Agent | dsl.Application | dsl.Flow | list
+# Configuration dictionary type
+ConfigDict = dict[str, Any]
+
+
+# ---------------- Shared Base Types and Enums ----------------
+
+
+class PrimitiveTypeEnum(str, Enum):
+    """Represents the type of data a user or system input can accept within the DSL."""
+
+    audio = "audio"
+    boolean = "boolean"
+    bytes = "bytes"
+    citation_document = "citation_document"
+    citation_url = "citation_url"
+    date = "date"
+    datetime = "datetime"
+    int = "int"
+    file = "file"
+    float = "float"
+    image = "image"
+    text = "text"
+    time = "time"
+    video = "video"
+    thinking = "thinking"
+
+
+class StepCardinality(str, Enum):
+    """Does this step emit 1 (one) or 0...N (many) items?"""
+
+    one = "one"
+    many = "many"
+
+
+ReferenceT = TypeVar("ReferenceT")
+
+
+class Reference(BaseModel, Generic[ReferenceT]):
+    """Represents a reference to another component by its ID."""
+
+    # model_config = PydanticConfigDict(extra="forbid")
+
+    ref: str = Field(..., alias="$ref")
+
+
+def _contains_reference_and_str(type_hint: Any) -> bool:
+    """Check if type contains both Reference and str in a union."""
+    # Get union args (handles Union, | syntax, and Optional)
+    origin = get_origin(type_hint)
+    if origin not in (Union, None) and not isinstance(
+        type_hint, types.UnionType
+    ):
+        return False
+
+    args = get_args(type_hint)
+    if not args:
+        return False
+
+    has_str = str in args
+    has_ref = any(
+        get_origin(arg) is Reference
+        or (hasattr(arg, "__mro__") and Reference in arg.__mro__)
+        for arg in args
+    )
+    return has_str and has_ref
+
+
+def _should_transform_field(type_hint: Any) -> tuple[bool, bool]:
+    """
+    Check if field should be transformed.
+    Returns: (should_transform, is_list)
+    """
+    # Check direct union: Reference[T] | str
+    if _contains_reference_and_str(type_hint):
+        return True, False
+
+    # Check list of union: list[Reference[T] | str]
+    origin = get_origin(type_hint)
+    if origin is list:
+        args = get_args(type_hint)
+        if args and _contains_reference_and_str(args[0]):
+            return True, True
+
+    # Check optional list: list[Reference[T] | str] | None
+    if origin is Union or isinstance(type_hint, types.UnionType):
+        for arg in get_args(type_hint):
+            if get_origin(arg) is list:
+                inner_args = get_args(arg)
+                if inner_args and _contains_reference_and_str(inner_args[0]):
+                    return True, True
+
+    return False, False
+
+
+class StrictBaseModel(BaseModel):
+    """Base model with extra fields forbidden."""
+
+    model_config = PydanticConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_string_references(cls, data: Any) -> Any:
+        """
+        Normalize string references to Reference objects before validation.
+
+        Transforms:
+        - `field: "ref_id"` -> `field: {"$ref": "ref_id"}`
+        - `field: ["ref1", "ref2"]` -> `field: [{"$ref": "ref1"}, {"$ref": "ref2"}]`
+
+        Only applies to fields typed as `Reference[T] | str` or `list[Reference[T] | str]`.
+        """
+        if not isinstance(data, dict):
+            return data
+
+        # Get type hints (evaluates ForwardRefs)
+        hints = typing.get_type_hints(cls)
+
+        # Transform fields
+        for field_name, field_value in data.items():
+            if field_name == "type" or field_name not in hints:
+                continue
+
+            should_transform, is_list = _should_transform_field(
+                hints[field_name]
+            )
+            if not should_transform:
+                continue
+
+            if is_list and isinstance(field_value, list):
+                data[field_name] = [
+                    {"$ref": item} if isinstance(item, str) else item
+                    for item in field_value
+                ]
+            elif not is_list and isinstance(field_value, str):
+                data[field_name] = {"$ref": field_value}
+
+        return data
+
+
+class BatchConfig(BaseModel):
+    """Configuration for batch execution.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    batch_size: int = Field(
+        default=25,
+        description="Max number of rows to send to a step at a time",
+        gt=0,
+    )
+
+
+class ConcurrencyConfig(BaseModel):
+    """Configuration for concurrent processing.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    num_workers: int = Field(
+        default=1,
+        description="Number of async workers for batch operations",
+        gt=0,
+    )
+
+
+class BatchableStepMixin(BaseModel):
+    """A mixin for steps that support concurrent batch processing."""
+
+    batch_config: BatchConfig = Field(
+        default_factory=BatchConfig,
+        description="Configuration for processing the input stream in batches. If omitted, the step processes items one by one.",
+    )
+
+
+class CacheConfig(BaseModel):
+    directory: PathLike = Field(
+        default=pathlib.Path("./.qtype-cache"),
+        description="Base cache directory.",
+    )
+    namespace: Optional[str] = Field(
+        default=None, description="Logical namespace for cache keys."
+    )
+    on_error: Literal["Cache", "Drop"] = "Drop"
+    version: str = Field(
+        default="1.0", description="Bump to invalidate old cache."
+    )
+    compress: bool = Field(default=False, description="Compress stored data.")
+    ttl: Optional[int] = Field(
+        default=None, description="Optional time-to-live in seconds."
+    )
+
+
+class CachedStepMixin(BaseModel):
+    """A mixin for steps that support caching."""
+
+    cache_config: CacheConfig | None = Field(
+        default=None,
+        description="Configuration for caching step outputs. If omitted, caching is disabled.",
+    )
+
+
+class ConcurrentStepMixin(BaseModel):
+    """A mixin for steps that support concurrent processing."""
+
+    concurrency_config: ConcurrencyConfig = Field(
+        default_factory=ConcurrencyConfig,
+        description="Configuration for processing the input stream concurrently. If omitted, the step processes items sequentially.",
+    )

qtype/cli.py

@@ -7,6 +7,10 @@ import importlib
 import logging
 from pathlib import Path
 
+from qtype.base.logging import get_logger
+
+logger = get_logger("application.facade")
+
 try:
     from importlib.metadata import entry_points
 except ImportError:

qtype/commands/convert.py

@@ -8,12 +8,26 @@ import argparse
 import logging
 from pathlib import Path
 
-from qtype.application.facade import QTypeFacade
-from qtype.dsl.model import Application, ToolList
+from qtype.dsl.model import Application, Document, ToolList
 
 logger = logging.getLogger(__name__)
 
 
+def _convert_to_yaml(doc: Application | ToolList) -> str:
+    """Convert a document to YAML format."""
+    from pydantic_yaml import to_yaml_str
+
+    # Wrap in Document if needed
+    if isinstance(doc, Application):
+        wrapped = Document(root=doc)
+    else:
+        wrapped = doc
+
+    # NOTE: We use exclude_none but NOT exclude_unset because discriminator
+    # fields like 'type' have default values and must be included in output
+    return to_yaml_str(wrapped, exclude_none=True)
+
+
 def convert_api(args: argparse.Namespace) -> None:
     """Convert API specification to qtype format."""
     from qtype.application.converters.tools_from_api import tools_from_api
@@ -36,9 +50,8 @@ def convert_api(args: argparse.Namespace) -> None:
                 types=types,
                 auths=auths,
             )
-        # Use facade to convert to YAML format
-        facade = QTypeFacade()
-        content = facade.convert_document(doc)
+        # Convert to YAML format
+        content = _convert_to_yaml(doc)
 
         # Write to file or stdout
         if args.output:
@@ -79,9 +92,8 @@ def convert_module(args: argparse.Namespace) -> None:
                 root=list(tools),
             )
 
-        # Use facade to convert to YAML format
-        facade = QTypeFacade()
-        content = facade.convert_document(doc)
+        # Convert to YAML format
+        content = _convert_to_yaml(doc)
 
         # Write to file or stdout
         if args.output: