qtype 0.0.12__py3-none-any.whl → 0.1.7__py3-none-any.whl

qtype/application/facade.py

@@ -2,69 +2,96 @@
 
 from __future__ import annotations
 
+import logging
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-from pydantic import BaseModel
-
-from qtype.base.logging import get_logger
-from qtype.base.types import CustomTypeRegistry, DocumentRootType, PathLike
-from qtype.dsl.base_types import StepCardinality
-from qtype.dsl.model import Application as DSLApplication
-from qtype.dsl.model import DocumentType
-from qtype.interpreter.batch.types import BatchConfig
+from qtype.base.types import PathLike
 from qtype.semantic.model import Application as SemanticApplication
-from qtype.semantic.model import Variable
+from qtype.semantic.model import DocumentType as SemanticDocumentType
+
+# Note: There should be _zero_ imports here at the top that import qtype.interpreter.
+# That's the whole point of this facade - to avoid importing optional
+# dependencies unless these methods are called.
 
-logger = get_logger("application.facade")
+logger = logging.getLogger(__name__)
 
 
 class QTypeFacade:
     """
-    Simplified interface for all qtype operations.
+    Simplified interface for qtype operations.
 
-    This facade hides the complexity of coordinating between DSL, semantic,
-    and interpreter layers, providing a clean API for common operations.
+    This facade provides lazy-loading wrappers for operations that require
+    optional dependencies (interpreter package), allowing base qtype to work
+    without those dependencies installed.
     """
 
-    def load_dsl_document(
-        self, path: PathLike
-    ) -> tuple[DocumentRootType, CustomTypeRegistry]:
-        from qtype.loader import load_document
+    def telemetry(self, spec: SemanticDocumentType) -> None:
+        from qtype.interpreter.telemetry import register
 
-        return load_document(Path(path).read_text(encoding="utf-8"))
+        if isinstance(spec, SemanticApplication) and spec.telemetry:
+            logger.info(
+                f"Telemetry enabled with endpoint: {spec.telemetry.endpoint}"
+            )
+            # Register telemetry if needed
+            register(spec.telemetry, self.secret_manager(spec), spec.id)
 
-    def load_and_validate(self, path: PathLike) -> DocumentRootType:
-        """Load and validate a document."""
-        logger.info("Document loaded, proceeding to validation")
-        root, _ = self.load_dsl_document(path)
-        return root
+    def secret_manager(self, spec: SemanticDocumentType):
+        """
+        Create a secret manager based on the specification.
 
-    def load_semantic_model(
-        self, path: PathLike
-    ) -> tuple[SemanticApplication, CustomTypeRegistry]:
-        """Load a document and return the resolved semantic model."""
-        from qtype.loader import load
+        Args:
+            spec: SemanticDocumentType specification
 
-        content = Path(path).read_text(encoding="utf-8")
-        return load(content)
+        Returns:
+            Secret manager instance
+        """
+        from qtype.interpreter.base.secrets import create_secret_manager
 
-    def execute_workflow(
+        if isinstance(spec, SemanticApplication):
+            return create_secret_manager(spec.secret_manager)
+        else:
+            raise ValueError(
+                "Can't create secret manager for non-Application spec"
+            )
+
+    async def execute_workflow(
         self,
         path: PathLike,
-        inputs: dict | pd.DataFrame,
+        inputs: dict | Any,
         flow_name: str | None = None,
-        batch_config: BatchConfig | None = None,
         **kwargs: Any,
-    ) -> pd.DataFrame | list[Variable]:
-        """Execute a complete workflow from document to results."""
-        logger.info(f"Executing workflow from {path}")
+    ) -> Any:
+        """
+        Execute a complete workflow from document to results.
+
+        Args:
+            path: Path to the QType specification file
+            inputs: Dictionary of input values or DataFrame for batch
+            flow_name: Optional name of flow to execute
+            **kwargs: Additional dependencies for execution
+
+        Returns:
+            DataFrame with results (one row per input)
+        """
+        import pandas as pd
+        from opentelemetry import trace
+
+        from qtype.interpreter.base.executor_context import ExecutorContext
+        from qtype.interpreter.converters import (
+            dataframe_to_flow_messages,
+            flow_messages_to_dataframe,
+        )
+        from qtype.interpreter.flow import run_flow
+        from qtype.interpreter.types import Session
+        from qtype.semantic.loader import load
 
         # Load the semantic application
-        semantic_model, type_registry = self.load_semantic_model(path)
+        semantic_model, type_registry = load(Path(path))
+        assert isinstance(semantic_model, SemanticApplication)
+        self.telemetry(semantic_model)
 
-        # Find the flow to execute (inlined from _find_flow)
+        # Find the flow to execute
         if flow_name:
             target_flow = None
             for flow in semantic_model.flows:
@@ -78,57 +105,47 @@ class QTypeFacade:
                 target_flow = semantic_model.flows[0]
             else:
                 raise ValueError("No flows found in application")
-        if target_flow.cardinality == StepCardinality.many:
-            if isinstance(inputs, dict):
-                inputs = pd.DataFrame([inputs])
-            if not isinstance(inputs, pd.DataFrame):
-                raise ValueError(
-                    "Input must be a DataFrame for flows with 'many' cardinality"
-                )
-            from qtype.interpreter.batch.flow import batch_execute_flow
-
-            batch_config = batch_config or BatchConfig()
-            results, errors = batch_execute_flow(
-                target_flow, inputs, batch_config, **kwargs
-            )  # type: ignore
-            return results
-        else:
-            from qtype.interpreter.flow import execute_flow
-
-            args = {**kwargs, **inputs}
-            return execute_flow(target_flow, **args)
 
-    def visualize_application(self, path: PathLike) -> str:
-        """Visualize an application as Mermaid diagram."""
-        from qtype.semantic.visualize import visualize_application
+        logger.info(f"Executing flow {target_flow.id} from {path}")
 
-        semantic_model, _ = self.load_semantic_model(path)
-        return visualize_application(semantic_model)
+        # Convert inputs to DataFrame (normalize single dict to 1-row DataFrame)
 
-    def convert_document(self, document: DocumentType) -> str:
-        """Convert a document to YAML format."""
-        # Wrap DSLApplication in Document if needed
-        wrapped_document: BaseModel = document
-        if isinstance(document, DSLApplication):
-            from qtype.dsl.model import Document
-
-            wrapped_document = Document(root=document)
-
-        # Try to use pydantic_yaml first
-        try:
-            from pydantic_yaml import to_yaml_str
-
-            return to_yaml_str(
-                wrapped_document, exclude_unset=True, exclude_none=True
+        if isinstance(inputs, dict):
+            input_df = pd.DataFrame([inputs])
+        elif isinstance(inputs, pd.DataFrame):
+            input_df = inputs
+        else:
+            raise ValueError(
+                f"Inputs must be dict or DataFrame, got {type(inputs)}"
             )
-        except ImportError:
-            # Fallback to basic YAML if pydantic_yaml is not available
-            import yaml
 
-            document_dict = wrapped_document.model_dump(
-                exclude_unset=True, exclude_none=True
-            )
-            return yaml.dump(document_dict, default_flow_style=False)
+        # Create session
+        session = Session(
+            session_id=kwargs.pop("session_id", "default"),
+            conversation_history=kwargs.pop("conversation_history", []),
+        )
+
+        # Convert DataFrame to FlowMessages
+        initial_messages = dataframe_to_flow_messages(input_df, session)
+
+        # Execute the flow
+        secret_manager = self.secret_manager(semantic_model)
+
+        context = ExecutorContext(
+            secret_manager=secret_manager,
+            tracer=trace.get_tracer(__name__),
+        )
+        results = await run_flow(
+            target_flow,
+            initial_messages,
+            context=context,
+            **kwargs,
+        )
+
+        # Convert results back to DataFrame
+        results_df = flow_messages_to_dataframe(results, target_flow)
+
+        return results_df
 
     def generate_aws_bedrock_models(self) -> list[dict[str, Any]]:
         """

qtype/base/types.py

@@ -3,11 +3,24 @@
 from __future__ import annotations
 
 import pathlib
-from typing import Any, Type, Union
+import types
+import typing
+from enum import Enum
+from typing import (
+    Any,
+    Generic,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
 
 from pydantic import BaseModel
-
-from qtype.dsl import model as dsl
+from pydantic import ConfigDict as PydanticConfigDict
+from pydantic import Field, model_validator
 
 # JSON-serializable value types
 JSONValue = Union[
@@ -20,10 +33,217 @@ JSONValue = Union[
     list["JSONValue"],
 ]
 
-# Configuration dictionary type
-ConfigDict = dict[str, Any]
-
 # Path-like type (string or Path object)
 PathLike = Union[str, pathlib.Path]
+
 CustomTypeRegistry = dict[str, Type[BaseModel]]
-DocumentRootType = dsl.Agent | dsl.Application | dsl.Flow | list
+# Configuration dictionary type
+ConfigDict = dict[str, Any]
+
+
+# ---------------- Shared Base Types and Enums ----------------
+
+
+class PrimitiveTypeEnum(str, Enum):
+    """Represents the type of data a user or system input can accept within the DSL."""
+
+    audio = "audio"
+    boolean = "boolean"
+    bytes = "bytes"
+    citation_document = "citation_document"
+    citation_url = "citation_url"
+    date = "date"
+    datetime = "datetime"
+    int = "int"
+    file = "file"
+    float = "float"
+    image = "image"
+    text = "text"
+    time = "time"
+    video = "video"
+    thinking = "thinking"
+
+
+class StepCardinality(str, Enum):
+    """Does this step emit 1 (one) or 0...N (many) items?"""
+
+    one = "one"
+    many = "many"
+
+
+ReferenceT = TypeVar("ReferenceT")
+
+
+class Reference(BaseModel, Generic[ReferenceT]):
+    """Represents a reference to another component by its ID."""
+
+    # model_config = PydanticConfigDict(extra="forbid")
+
+    ref: str = Field(..., alias="$ref")
+
+
+def _contains_reference_and_str(type_hint: Any) -> bool:
+    """Check if type contains both Reference and str in a union."""
+    # Get union args (handles Union, | syntax, and Optional)
+    origin = get_origin(type_hint)
+    if origin not in (Union, None) and not isinstance(
+        type_hint, types.UnionType
+    ):
+        return False
+
+    args = get_args(type_hint)
+    if not args:
+        return False
+
+    has_str = str in args
+    has_ref = any(
+        get_origin(arg) is Reference
+        or (hasattr(arg, "__mro__") and Reference in arg.__mro__)
+        for arg in args
+    )
+    return has_str and has_ref
+
+
+def _should_transform_field(type_hint: Any) -> tuple[bool, bool]:
+    """
+    Check if field should be transformed.
+    Returns: (should_transform, is_list)
+    """
+    # Check direct union: Reference[T] | str
+    if _contains_reference_and_str(type_hint):
+        return True, False
+
+    # Check list of union: list[Reference[T] | str]
+    origin = get_origin(type_hint)
+    if origin is list:
+        args = get_args(type_hint)
+        if args and _contains_reference_and_str(args[0]):
+            return True, True
+
+    # Check optional list: list[Reference[T] | str] | None
+    if origin is Union or isinstance(type_hint, types.UnionType):
+        for arg in get_args(type_hint):
+            if get_origin(arg) is list:
+                inner_args = get_args(arg)
+                if inner_args and _contains_reference_and_str(inner_args[0]):
+                    return True, True
+
+    return False, False
+
+
+class StrictBaseModel(BaseModel):
+    """Base model with extra fields forbidden."""
+
+    model_config = PydanticConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_string_references(cls, data: Any) -> Any:
+        """
+        Normalize string references to Reference objects before validation.
+
+        Transforms:
+        - `field: "ref_id"` -> `field: {"$ref": "ref_id"}`
+        - `field: ["ref1", "ref2"]` -> `field: [{"$ref": "ref1"}, {"$ref": "ref2"}]`
+
+        Only applies to fields typed as `Reference[T] | str` or `list[Reference[T] | str]`.
+        """
+        if not isinstance(data, dict):
+            return data
+
+        # Get type hints (evaluates ForwardRefs)
+        hints = typing.get_type_hints(cls)
+
+        # Transform fields
+        for field_name, field_value in data.items():
+            if field_name == "type" or field_name not in hints:
+                continue
+
+            should_transform, is_list = _should_transform_field(
+                hints[field_name]
+            )
+            if not should_transform:
+                continue
+
+            if is_list and isinstance(field_value, list):
+                data[field_name] = [
+                    {"$ref": item} if isinstance(item, str) else item
+                    for item in field_value
+                ]
+            elif not is_list and isinstance(field_value, str):
+                data[field_name] = {"$ref": field_value}
+
+        return data
+
+
+class BatchConfig(BaseModel):
+    """Configuration for batch execution.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    batch_size: int = Field(
+        default=25,
+        description="Max number of rows to send to a step at a time",
+        gt=0,
+    )
+
+
+class ConcurrencyConfig(BaseModel):
+    """Configuration for concurrent processing.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    num_workers: int = Field(
+        default=1,
+        description="Number of async workers for batch operations",
+        gt=0,
+    )
+
+
+class BatchableStepMixin(BaseModel):
+    """A mixin for steps that support concurrent batch processing."""
+
+    batch_config: BatchConfig = Field(
+        default_factory=BatchConfig,
+        description="Configuration for processing the input stream in batches. If omitted, the step processes items one by one.",
+    )
+
+
+class CacheConfig(BaseModel):
+    directory: PathLike = Field(
+        default=pathlib.Path("./.qtype-cache"),
+        description="Base cache directory.",
+    )
+    namespace: Optional[str] = Field(
+        default=None, description="Logical namespace for cache keys."
+    )
+    on_error: Literal["Cache", "Drop"] = "Drop"
+    version: str = Field(
+        default="1.0", description="Bump to invalidate old cache."
+    )
+    compress: bool = Field(default=False, description="Compress stored data.")
+    ttl: Optional[int] = Field(
+        default=None, description="Optional time-to-live in seconds."
+    )
+
+
+class CachedStepMixin(BaseModel):
+    """A mixin for steps that support caching."""
+
+    cache_config: CacheConfig | None = Field(
+        default=None,
+        description="Configuration for caching step outputs. If omitted, caching is disabled.",
+    )
+
+
+class ConcurrentStepMixin(BaseModel):
+    """A mixin for steps that support concurrent processing."""
+
+    concurrency_config: ConcurrencyConfig = Field(
+        default_factory=ConcurrencyConfig,
+        description="Configuration for processing the input stream concurrently. If omitted, the step processes items sequentially.",
+    )

qtype/cli.py

@@ -7,6 +7,10 @@ import importlib
 import logging
 from pathlib import Path
 
+from qtype.base.logging import get_logger
+
+logger = get_logger("application.facade")
+
 try:
     from importlib.metadata import entry_points
 except ImportError:
@@ -131,7 +135,7 @@ def main() -> None:
     # Set logging level based on user input
     logging.basicConfig(
         level=getattr(logging, args.log_level),
-        format="%(levelname)s: %(message)s",
+        format="%(asctime)s - %(levelname)s: %(message)s",
     )
 
     # Dispatch to the selected subcommand

qtype/commands/convert.py

@@ -8,14 +8,62 @@ import argparse
 import logging
 from pathlib import Path
 
-from qtype.application.facade import QTypeFacade
+from qtype.dsl.model import Application, Document, ToolList
 
 logger = logging.getLogger(__name__)
 
 
+def _convert_to_yaml(doc: Application | ToolList) -> str:
+    """Convert a document to YAML format."""
+    from pydantic_yaml import to_yaml_str
+
+    # Wrap in Document if needed
+    if isinstance(doc, Application):
+        wrapped = Document(root=doc)
+    else:
+        wrapped = doc
+
+    # NOTE: We use exclude_none but NOT exclude_unset because discriminator
+    # fields like 'type' have default values and must be included in output
+    return to_yaml_str(wrapped, exclude_none=True)
+
+
 def convert_api(args: argparse.Namespace) -> None:
     """Convert API specification to qtype format."""
-    raise NotImplementedError("API conversion is not implemented yet.")
+    from qtype.application.converters.tools_from_api import tools_from_api
+
+    try:
+        api_name, auths, tools, types = tools_from_api(args.api_spec)
+        if not tools:
+            raise ValueError(
+                f"No tools found from the API specification: {args.api_spec}"
+            )
+        if not auths and not types:
+            doc = ToolList(
+                root=list(tools),
+            )
+        else:
+            doc: Application | ToolList = Application(
+                id=api_name,
+                description=f"Tools created from API specification {args.api_spec}",
+                tools=list(tools),
+                types=types,
+                auths=auths,
+            )
+        # Convert to YAML format
+        content = _convert_to_yaml(doc)
+
+        # Write to file or stdout
+        if args.output:
+            output_path = Path(args.output)
+            output_path.write_text(content, encoding="utf-8")
+            logger.info(f"✅ Converted tools saved to {output_path}")
+        else:
+            print(content)
+
+    except Exception as e:
+        logger.error(f"❌ Conversion failed: {e}")
+        raise
 
 
 def convert_module(args: argparse.Namespace) -> None:
@@ -23,7 +71,6 @@ def convert_module(args: argparse.Namespace) -> None:
     from qtype.application.converters.tools_from_module import (
         tools_from_module,
     )
-    from qtype.dsl.model import Application, ToolList
 
     try:
         tools, types = tools_from_module(args.module_path)
@@ -45,9 +92,8 @@ def convert_module(args: argparse.Namespace) -> None:
                 root=list(tools),
             )
 
-        # Use facade to convert to YAML format
-        facade = QTypeFacade()
-        content = facade.convert_document(doc)
+        # Convert to YAML format
+        content = _convert_to_yaml(doc)
 
         # Write to file or stdout
         if args.output:

qtype/commands/generate.py

@@ -84,8 +84,44 @@ def generate_schema(args: argparse.Namespace) -> None:
             'output' attribute specifying the output file path.
     """
     schema = Document.model_json_schema()
+
     # Add the $schema property to indicate JSON Schema version
     schema["$schema"] = "http://json-schema.org/draft-07/schema#"
+
+    # Add custom YAML tag definitions for QType loader features
+    if "$defs" not in schema:
+        schema["$defs"] = {}
+
+    # Note: Custom YAML tags (!include, !include_raw) and environment variable
+    # substitution (${VAR}) are handled by the QType YAML loader at parse time,
+    # not by JSON Schema validation. We define them in $defs for documentation
+    # purposes, but we don't apply them to string fields since:
+    # 1. They would cause false positives (e.g., "localhost" matching as valid)
+    # 2. The YAML loader processes these before schema validation occurs
+    # 3. After YAML loading, the schema sees the resolved/substituted values
+    #
+    # Schema validation happens on the post-processed document structure,
+    # so we don't need to (and shouldn't) validate the raw YAML tag syntax.
+
+    # Define custom YAML tags used by QType loader
+    schema["$defs"]["qtype_include_tag"] = {
+        "type": "string",
+        "pattern": "^!include\\s+.+",
+        "description": "Include external YAML file using QType's !include tag",
+    }
+
+    schema["$defs"]["qtype_include_raw_tag"] = {
+        "type": "string",
+        "pattern": "^!include_raw\\s+.+",
+        "description": "Include raw text file using QType's !include_raw tag",
+    }
+
+    schema["$defs"]["qtype_env_var"] = {
+        "type": "string",
+        "pattern": "^.*\\$\\{[^}:]+(?::[^}]*)?\\}.*$",
+        "description": "String with environment variable substitution using ${VAR_NAME} or ${VAR_NAME:default} syntax",
+    }
+
     output = json.dumps(schema, indent=2)
     output_path: Optional[str] = getattr(args, "output", None)
     if output_path:
@@ -150,6 +186,14 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
         import networkx  # noqa: F401
         import ruff  # type: ignore[import-untyped]  # noqa: F401
 
+        has_semantic_deps = True
+    except ImportError:
+        logger.debug(
+            "NetworkX or Ruff is not installed. Skipping semantic model generation."
+        )
+        has_semantic_deps = False
+
+    if has_semantic_deps:
         from qtype.semantic.generate import generate_semantic_model
 
         semantic_parser = generate_subparsers.add_parser(
@@ -164,7 +208,3 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
             help="Output file for the semantic model (default: stdout)",
         )
         semantic_parser.set_defaults(func=generate_semantic_model)
-    except ImportError:
-        logger.debug(
-            "NetworkX or Ruff is not installed. Skipping semantic model generation."
-        )