qtype 0.1.11__py3-none-any.whl → 0.1.13__py3-none-any.whl

qtype/docs/skills/architect/references/patterns.md

@@ -0,0 +1,29 @@
+# QType Examples
+
+## Pattern: Simple Chatbot
+
+Basic conversational interface with memory.
+
+```yaml
+--8<-- "../examples/conversational_ai/simple_chatbot.qtype.yaml"
+```
+
+---
+
+## Pattern: Tool-Based Research
+
+Web search with synthesis using external tools.
+
+```yaml
+--8<-- "../examples/research_assistant/research_assistant.qtype.yaml"
+```
+
+---
+
+## Pattern: Data Pipeline with LLM Processing
+
+Parallel processing of database records with LLM analysis.
+
+```yaml
+--8<-- "../examples/data_processing/dataflow_pipelines.qtype.yaml"
+```

qtype/docs/stylesheets/extra.css

@@ -0,0 +1,27 @@
+/* Custom styles for QType documentation */
+
+/* Increase max width of main content area */
+.md-grid {
+  max-width: 75rem;  /* Default is 61rem (976px), this increases to 1280px */
+}
+
+/* Adjust content width for better readability */
+.md-content {
+  max-width: none;
+}
+
+/* Ensure sidebars remain proportional */
+.md-sidebar--primary {
+  left: 0;
+}
+
+.md-sidebar--secondary {
+  right: 0;
+}
+
+/* Make section headers darker gray */
+.md-nav__title,
+.md-nav__item--section > .md-nav__link {
+  color: #424242 !important;
+  font-weight: 600;
+}

qtype/dsl/custom_types.py

@@ -43,7 +43,8 @@ def build_dynamic_types(
             resolved_type = ForwardRef(type_str)
 
         if is_optional:
-            return Union[resolved_type, None], True
+            # Type checker: resolved_type is runtime-constructed type
+            return Union[resolved_type, None], True  # type: ignore[valid-type]
 
         return resolved_type, False
 

qtype/dsl/linker.py

@@ -253,18 +253,26 @@ def _resolve_rootmodel_references(
         model: RootModel instance to resolve
         lookup_map: Map of component IDs to objects
     """
-    root_list = model.root  # type: ignore
-    if not isinstance(root_list, list):
-        return
+    root_value = model.root  # type: ignore[attr-defined]
 
-    for i, item in enumerate(root_list):
+    def _resolve_root_item(item: Any) -> Any:
         match item:
             case base_types.Reference():
-                root_list[i] = _resolve_reference(
-                    item.ref, type(item), lookup_map
-                )
+                return _resolve_reference(item.ref, type(item), lookup_map)
             case BaseModel():
                 _resolve_all_references(item, lookup_map)
+                return item
+            case _:
+                return item
+
+    # RootModel can wrap either a list (e.g., ToolList) or a single model
+    # (e.g., Document -> Application). We need to traverse both.
+    if isinstance(root_value, list):
+        for i, item in enumerate(root_value):
+            root_value[i] = _resolve_root_item(item)
+        return
+    else:
+        model.root = _resolve_root_item(root_value)  # type: ignore[attr-defined]
 
 
 def _resolve_list_references(
@@ -283,6 +291,10 @@ def _resolve_list_references(
                 field_value[i] = _resolve_reference(
                     item.ref, type(item), lookup_map
                 )
+            case str():
+                # Resolve string IDs to their objects (e.g., variable IDs to Variables)
+                if item in lookup_map:
+                    field_value[i] = lookup_map[item]
             case BaseModel():
                 _resolve_all_references(item, lookup_map)
 
@@ -301,6 +313,10 @@ def _resolve_dict_references(
         match v:
             case base_types.Reference():
                 field_value[k] = _resolve_reference(v.ref, type(v), lookup_map)
+            case str():
+                # Resolve string IDs to their objects (e.g., variable IDs to Variables)
+                if v in lookup_map:
+                    field_value[k] = lookup_map[v]
             case BaseModel():
                 _resolve_all_references(v, lookup_map)
 

qtype/dsl/loader.py

@@ -6,7 +6,7 @@ This module provides two explicit functions for loading YAML:
 - load_yaml_string(content, base_path): Load YAML from a string
 
 Both support:
-- Environment variable substitution (${VAR} or ${VAR:default})
+- Environment variable substitution (${VAR} or ${VAR:-default})
 - File inclusion (!include and !include_raw)
 - Multiple URI schemes via fsspec (local, http, s3, etc.)
 
@@ -81,7 +81,7 @@ def _substitute_env_vars(value: str) -> str:
     """
     Substitute environment variables in a string.
 
-    Supports ${VAR_NAME} or ${VAR_NAME:default} syntax.
+    Supports ${VAR_NAME} or ${VAR_NAME:-default} syntax.
 
     Args:
         value: String containing environment variable references
@@ -92,7 +92,7 @@ def _substitute_env_vars(value: str) -> str:
     Raises:
         ValueError: If required environment variable is not found
     """
-    pattern = r"\$\{([^}:]+)(?::([^}]*))?\}"
+    pattern = r"\$\{([^}:]+)(?::-([^}]*))?\}"
 
     def replace_env_var(match: re.Match[str]) -> str:
         var_name = match.group(1)

qtype/dsl/model.py

@@ -79,6 +79,11 @@ def _resolve_list_type(
             if cls == element_type:
                 return ListType(element_type=name)
         return ListType(element_type=str(element_type))
+    elif isinstance(element_type, type) and issubclass(
+        element_type, BaseModel
+    ):
+        # Custom type class - store its name as string reference
+        return ListType(element_type=element_type.__name__)
     else:
         raise ValueError(
             (
@@ -140,6 +145,8 @@ def _resolve_variable_type(
     Resolve a type to its corresponding representation.
 
     Handles primitive types, list types, domain types, and custom types.
+    Unknown types are returned as strings (forward references) and will be
+    validated later during the linking phase.
 
     Args:
         parsed_type: The type to resolve (can be string or already resolved)
@@ -172,17 +179,26 @@ def _resolve_variable_type(
     if custom is not None:
         return custom
 
-    # If it's not any known type, return it as a string.
-    # This assumes it might be a forward reference to a custom type.
-    return parsed_type
+    # If it's not any known type, raise an error
+    available_types = (
+        f"primitive types ({', '.join([t.value for t in PrimitiveTypeEnum])}), "
+        f"domain types ({', '.join(DOMAIN_CLASSES.keys())})"
+    )
+    if custom_type_registry:
+        available_types += (
+            f", or custom types ({', '.join(custom_type_registry.keys())})"
+        )
+    raise ValueError(
+        f"Unknown type '{parsed_type}'. Must be one of: {available_types}"
+    )
 
 
 def _resolve_type_field_validator(data: Any, info: ValidationInfo) -> Any:
     """
     Shared validator for resolving 'type' fields in models.
 
-    This validator resolves string-based type references using the custom
-    type registry from the validation context.
+    This validator handles optional '?' syntax and resolves string-based
+    type references using the custom type registry from the validation context.
 
     Args:
         data: The data dict being validated
@@ -196,6 +212,13 @@ def _resolve_type_field_validator(data: Any, info: ValidationInfo) -> Any:
         and "type" in data
         and isinstance(data["type"], str)
     ):
+        # Handle '?' suffix for optional types BEFORE type resolution
+        type_value = data["type"]
+        if type_value.endswith("?"):
+            # Strip '?' and mark as optional
+            data["type"] = type_value[:-1]
+            data["optional"] = True
+
         # Get the registry of custom types from the validation context.
         custom_types = (info.context or {}).get("custom_types", {})
         resolved = _resolve_variable_type(data["type"], custom_types)
@@ -203,6 +226,54 @@ def _resolve_type_field_validator(data: Any, info: ValidationInfo) -> Any:
     return data
 
 
+def _merge_vars_from_bindings(
+    existing: list[Reference[Variable] | str],
+    bindings: dict[str, Reference[Variable] | str],
+) -> list[Reference[Variable] | str]:
+    """Merge existing variables with bindings and deduplicate by variable ID.
+
+    Args:
+        existing: Existing list of variable references or IDs
+        bindings: Dict mapping parameter names to variable references or IDs
+
+    Returns:
+        Merged list with duplicates removed, preserving original form
+    """
+
+    def get_id(item):
+        return item.ref if isinstance(item, Reference) else item
+
+    seen = set()
+    result = []
+    for item in list(existing) + list(bindings.values()):
+        var_id = get_id(item)
+        if var_id not in seen:
+            seen.add(var_id)
+            result.append(item)
+    return result
+
+
+def _type_to_string(type_value: Any) -> str:
+    """Convert a type value to its string representation.
+
+    Args:
+        type_value: The type value to convert
+
+    Returns:
+        String representation of the type
+    """
+    if isinstance(type_value, str):
+        return type_value
+    elif isinstance(type_value, PrimitiveTypeEnum):
+        return type_value.value
+    elif isinstance(type_value, ListType):
+        return str(type_value)
+    elif isinstance(type_value, type):
+        return type_value.__name__
+    else:
+        return str(type_value)
+
+
 class Variable(StrictBaseModel):
     """Schema for a variable that can serve as input, output, or parameter within the DSL."""
 
@@ -216,15 +287,37 @@ class Variable(StrictBaseModel):
             "Type of data expected or produced. Either a CustomType or domain specific type."
         ),
     )
+    optional: bool = Field(
+        default=False,
+        description=(
+            "Whether this variable can be unset or None. "
+            "Use '?' suffix in type string as shorthand (e.g., 'text?')."
+        ),
+    )
 
-    ui: UIType | None = Field(None, description="Hints for the UI if needed.")
+    ui: UIType | None = Field(
+        default=None, description="Hints for the UI if needed."
+    )
 
     @model_validator(mode="before")
     @classmethod
     def resolve_type(cls, data: Any, info: ValidationInfo) -> Any:
-        """Resolve string-based type references using the shared validator."""
+        """Resolve string-based type references and handle optional '?' syntax."""
         return _resolve_type_field_validator(data, info)
 
+    @model_serializer
+    def serialize_model(self):
+        """Serialize with '?' suffix for optional types."""
+        result: dict[str, Any] = {"id": self.id}
+
+        type_str = _type_to_string(self.type)
+        result["type"] = f"{type_str}?" if self.optional else type_str
+
+        if self.ui is not None:
+            result["ui"] = self.ui.model_dump()
+
+        return result
+
     @model_validator(mode="after")
     def validate_ui_type(self) -> Variable:
         """Ensure at least one credential source is provided."""
@@ -260,36 +353,6 @@ class CustomType(StrictBaseModel):
     properties: dict[str, str]
 
 
-class ToolParameter(BaseModel):
-    """Defines a tool input or output parameter with type and optional flag."""
-
-    type: VariableType | str
-    optional: bool = Field(
-        default=False, description="Whether this parameter is optional"
-    )
-
-    @model_validator(mode="before")
-    @classmethod
-    def resolve_type(cls, data: Any, info: ValidationInfo) -> Any:
-        """Resolve string-based type references using the shared validator."""
-        return _resolve_type_field_validator(data, info)
-
-    @staticmethod
-    def _serialize_type(value):
-        if isinstance(value, type):
-            return value.__name__
-        elif hasattr(value, "__name__"):
-            return value.__name__
-        return value
-
-    @model_serializer
-    def _model_serializer(self):
-        # Use the default serialization, but ensure 'type' is a string
-        data = self.model_dump()
-        data["type"] = self._serialize_type(data.get("type"))
-        return data
-
-
 class ListType(BaseModel):
     """Represents a list type with a specific element type."""
 
@@ -417,11 +480,19 @@ class Construct(Step):
     """A step that converts variables into an instance of a Custom or Domain Type"""
 
     type: Literal["Construct"] = "Construct"
-    field_mapping: dict[str, str] = Field(
+    field_bindings: dict[str, Reference[Variable] | str] = Field(
         ...,
-        description="Mapping of type inputs to variable names, if needed.",
+        description="Mapping from type field names to flow variable names.",
     )
 
+    @model_validator(mode="after")
+    def infer_inputs_from_bindings(self) -> "Construct":
+        """Infer inputs from field bindings."""
+        self.inputs = _merge_vars_from_bindings(
+            self.inputs, self.field_bindings
+        )
+        return self
+
 
 class PromptTemplate(Step):
     """Defines a prompt template with a string format and variable bindings.
@@ -444,12 +515,12 @@ class Tool(StrictBaseModel, ABC):
     description: str = Field(
         ..., description="Description of what the tool does."
     )
-    inputs: dict[str, ToolParameter] = Field(
-        default_factory=dict,
+    inputs: list[Variable] = Field(
+        default_factory=list,
         description="Input parameters required by this tool.",
     )
-    outputs: dict[str, ToolParameter] = Field(
-        default_factory=dict,
+    outputs: list[Variable] = Field(
+        default_factory=list,
         description="Output parameters produced by this tool.",
     )
 
@@ -484,9 +555,9 @@ class APITool(Tool):
         default_factory=dict,
         description="Optional HTTP headers to include in the request.",
     )
-    parameters: dict[str, ToolParameter] = Field(
-        default_factory=dict,
-        description="Output parameters produced by this tool.",
+    parameters: list[Variable] = Field(
+        default_factory=list,
+        description="Path and query parameters for the API call.",
     )
 
 
@@ -621,6 +692,9 @@ class FieldExtractor(Step):
     The extracted data is used to construct the output variable by passing it
     as keyword arguments to the output type's constructor.
 
+    If there is no match and the output variable is optional, it is set to None.
+    If there is no match and the output variable is required, an error is raised.
+
     Example JSONPath expressions:
     - `$.field_name` - Extract a single field
     - `$.items[*]` - Extract all items from a list
@@ -632,10 +706,6 @@ class FieldExtractor(Step):
         ...,
         description="JSONPath expression to extract data from the input. Uses jsonpath-ng syntax.",
     )
-    fail_on_missing: bool = Field(
-        default=True,
-        description="Whether to raise an error if the JSONPath matches no data. If False, returns None.",
-    )
 
 
 class InvokeTool(Step, ConcurrentStepMixin):
@@ -647,15 +717,25 @@ class InvokeTool(Step, ConcurrentStepMixin):
         ...,
         description="Tool to invoke.",
     )
-    input_bindings: dict[str, str] = Field(
+    input_bindings: dict[str, Reference[Variable] | str] = Field(
         ...,
-        description="Mapping from variable references to tool input parameter names.",
+        description="Mapping from tool parameter names to flow variable names.",
     )
-    output_bindings: dict[str, str] = Field(
+    output_bindings: dict[str, Reference[Variable] | str] = Field(
         ...,
-        description="Mapping from variable references to tool output parameter names.",
+        description="Mapping from tool output names to flow variable names.",
     )
 
+    @model_validator(mode="after")
+    def infer_inputs_outputs_from_bindings(self) -> "InvokeTool":
+        self.inputs = _merge_vars_from_bindings(
+            self.inputs, self.input_bindings
+        )
+        self.outputs = _merge_vars_from_bindings(
+            self.outputs, self.output_bindings
+        )
+        return self
+
 
 class InvokeFlow(Step):
     """Invokes a flow with input and output bindings."""
@@ -666,15 +746,25 @@ class InvokeFlow(Step):
         ...,
         description="Flow to invoke.",
     )
-    input_bindings: dict[Reference[Variable], str] = Field(
+    input_bindings: dict[str, Reference[Variable] | str] = Field(
         ...,
-        description="Mapping from variable references to flow input variable IDs.",
+        description="Mapping from flow input variable IDs to step variable names.",
     )
-    output_bindings: dict[Reference[Variable], str] = Field(
+    output_bindings: dict[str, Reference[Variable] | str] = Field(
         ...,
-        description="Mapping from variable references to flow output variable IDs.",
+        description="Mapping from flow output variable IDs to step variable names.",
     )
 
+    @model_validator(mode="after")
+    def infer_inputs_outputs_from_bindings(self) -> "InvokeFlow":
+        self.inputs = _merge_vars_from_bindings(
+            self.inputs, self.input_bindings
+        )
+        self.outputs = _merge_vars_from_bindings(
+            self.outputs, self.output_bindings
+        )
+        return self
+
 
 #
 # ---------------- Secret Manager Component ----------------
@@ -979,6 +1069,21 @@ class FileSource(Source):
         description="Reference to a variable with an fsspec-compatible URI to read from, or the uri itself.",
     )
 
+    @model_validator(mode="after")
+    def infer_inputs_from_path(self) -> "FileSource":
+        """Add path variable to inputs if it's a variable reference."""
+        if isinstance(self.path, str):
+            # Path is a variable ID, add it to inputs
+            path_ref = Reference[Variable].model_validate({"$ref": self.path})
+            if path_ref not in self.inputs and self.path not in self.inputs:
+                self.inputs = list(self.inputs) + [path_ref]
+        elif isinstance(self.path, Reference):
+            # Path is already a Reference, add it to inputs
+            if self.path not in self.inputs:
+                self.inputs = list(self.inputs) + [self.path]
+        # If path is ConstantPath, don't add to inputs
+        return self
+
 
 class Writer(Step, BatchableStepMixin):
     """Base class for things that write data in batches."""
@@ -999,22 +1104,31 @@ class FileWriter(Writer, BatchableStepMixin):
         description="Configuration for processing the input stream in batches. If omitted, the step processes items one by one.",
     )
 
+    @model_validator(mode="after")
+    def infer_inputs_from_path(self) -> "FileWriter":
+        """Add path variable to inputs if it's a variable reference."""
+        if isinstance(self.path, str):
+            # Path is a variable ID, add it to inputs
+            path_ref = Reference[Variable].model_validate({"$ref": self.path})
+            if path_ref not in self.inputs and self.path not in self.inputs:
+                self.inputs = list(self.inputs) + [path_ref]
+        elif isinstance(self.path, Reference):
+            # Path is already a Reference, add it to inputs
+            if self.path not in self.inputs:
+                self.inputs = list(self.inputs) + [self.path]
+        # If path is ConstantPath, don't add to inputs
+        return self
+
 
 class Aggregate(Step):
     """
-    A terminal step that consumes an entire input stream and produces a single
-    summary message with success/error counts.
+    A step that, after all messages have been processed,
+    returns a single message containing the counts of successful and failed
+    messages. Other messages are passed through unchanged.
     """
 
     type: Literal["Aggregate"] = "Aggregate"
 
-    # Outputs are now optional. The user can provide 0, 1, 2, or 3 names.
-    # The order will be: success_count, error_count, total_count
-    outputs: list[Reference[Variable] | str] = Field(
-        default_factory=list,
-        description="References to the variables for the output. There should be one and only one output with type AggregateStats",
-    )
-
 
 #
 # ---------------- Retrieval Augmented Generation Components ----------------

qtype/examples/authentication/aws_authentication.qtype.yaml

@@ -0,0 +1,63 @@
+id: aws_auth_demo
+description: |
+  Demonstrates different methods of AWS authentication for Bedrock models.
+  Choose the method that best fits your deployment environment.
+
+# Method 1: AWS Profile (recommended for local development)
+# Uses credentials from ~/.aws/credentials
+auths:
+  - type: aws
+    id: aws_profile
+    profile_name: default
+    region: us-east-1
+
+# Method 2: Environment variables (recommended for production)
+# Set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION in environment
+# Uncomment to use:
+#  - type: aws
+#    id: aws_env
+#    access_key_id: ${AWS_ACCESS_KEY_ID}
+#    secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+#    region: us-east-1
+
+# Method 3: Role assumption (for cross-account access)
+# Uncomment and configure to use:
+#  - type: aws
+#    id: aws_role
+#    profile_name: base_profile
+#    role_arn: arn:aws:iam::123456789012:role/BedrockAccessRole
+#    role_session_name: qtype-app-session
+#    region: us-east-1
+
+models:
+  - type: Model
+    id: nova_micro
+    provider: aws-bedrock
+    model_id: us.amazon.nova-micro-v1:0
+    auth: aws_profile  # Change to aws_env, aws_keys, or aws_role as needed
+
+flows:
+  - id: simple_completion
+    variables:
+      - id: user_prompt
+        type: text
+      - id: formatted
+        type: text
+      - id: response
+        type: text
+    inputs:
+      - user_prompt
+    outputs:
+      - response
+    steps:
+      - type: PromptTemplate
+        id: format_prompt
+        template: "{user_prompt}"
+        inputs: [user_prompt]
+        outputs: [formatted]
+      
+      - type: LLMInference
+        id: generate
+        model: nova_micro
+        inputs: [formatted]
+        outputs: [response]

qtype/examples/conversational_ai/hello_world_chat.qtype.yaml

@@ -0,0 +1,43 @@
+id: hello_world
+description: A simple chat flow with OpenAI
+models:
+  - type: Model
+    id: gpt4
+    provider: openai
+    model_id: gpt-4
+    inference_params:
+      temperature: 0.7
+      max_tokens: 512
+    auth: openai_auth
+auths:
+  - type: api_key
+    id: openai_auth
+    api_key: ${OPENAI_KEY}
+    host: https://api.openai.com
+memories:
+  - id: chat_memory
+    token_limit: 10000
+flows:
+  - type: Flow
+    id: chat_example
+    interface:
+      type: Conversational
+    variables:
+      - id: user_message
+        type: ChatMessage
+      - id: response
+        type: ChatMessage
+    inputs:
+      - user_message
+    outputs:
+      - response
+    steps:
+      - id: llm_inference_step
+        type: LLMInference
+        model: gpt4
+        system_message: "You are a helpful assistant."
+        memory: chat_memory
+        inputs:
+          - user_message
+        outputs:
+          - response

qtype/examples/conversational_ai/simple_chatbot.qtype.yaml

@@ -0,0 +1,40 @@
+id: simple_chatbot
+description: A friendly chatbot with conversation memory using AWS Bedrock
+models:
+  - type: Model
+    id: nova_lite
+    provider: aws-bedrock
+    model_id: amazon.nova-lite-v1:0
+    inference_params:
+      temperature: 0.7
+      max_tokens: 512
+memories:
+  - id: conversation_memory
+    token_limit: 10000
+flows:
+  - type: Flow
+    id: chat_flow
+    interface:
+      type: Conversational
+    variables:
+      - id: user_message
+        type: ChatMessage
+      - id: assistant_response
+        type: ChatMessage
+    inputs:
+      - user_message
+    outputs:
+      - assistant_response
+    steps:
+      - id: generate_response
+        type: LLMInference
+        model: nova_lite
+        system_message: |
+          You are a friendly and helpful chatbot. You have a warm, conversational 
+          tone and enjoy helping users with their questions. You remember context 
+          from previous messages in the conversation.
+        memory: conversation_memory
+        inputs:
+          - user_message
+        outputs:
+          - assistant_response