qtype 0.1.7__py3-none-any.whl → 0.1.11__py3-none-any.whl

qtype/application/converters/tools_from_module.py

@@ -144,6 +144,11 @@ def _create_tool_from_function(
         for p in func_info["parameters"]
     }
 
+    # # quick hack
+    # for k, v in inputs.items():
+    #     if inspect.isclass(v.type) and issubclass(v.type, BaseModel):
+    #         v.type = str(v.type.__name__)
+
     # Create output parameter based on return type
     tool_id = func_info["module"] + "." + func_name
 
@@ -152,6 +157,7 @@ def _create_tool_from_function(
     )
 
     outputs = {"result": ToolParameter(type=output_type, optional=False)}
+    # outputs['result'].type =
 
     return PythonFunctionTool(
         id=tool_id,
@@ -264,6 +270,15 @@ def _map_python_type_to_variable_type(
     elif python_type in get_args(VariableType):
         # If it's a domain type, return its name
         return python_type  # type: ignore[no-any-return]
+    elif any(
+        [
+            (python_type is get_args(t)[0])
+            for t in get_args(VariableType)
+            if get_origin(t) is type
+        ]
+    ):
+        # It's the domain type, but the actual class (the user imported it)
+        return python_type.__name__
     elif inspect.isclass(python_type) and issubclass(python_type, BaseModel):
         # If it's a Pydantic model, create or retrieve its CustomType definition
         return _pydantic_to_custom_types(python_type, custom_types)

qtype/base/types.py

@@ -54,21 +54,13 @@ class PrimitiveTypeEnum(str, Enum):
     citation_url = "citation_url"
     date = "date"
     datetime = "datetime"
-    int = "int"
     file = "file"
     float = "float"
     image = "image"
+    int = "int"
     text = "text"
     time = "time"
     video = "video"
-    thinking = "thinking"
-
-
-class StepCardinality(str, Enum):
-    """Does this step emit 1 (one) or 0...N (many) items?"""
-
-    one = "one"
-    many = "many"
 
 
 ReferenceT = TypeVar("ReferenceT")

qtype/base/ui_shapes.py

@@ -0,0 +1,38 @@
+from enum import Enum
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import ConfigDict as PydanticConfigDict
+from pydantic import Field
+
+from qtype.base.types import PrimitiveTypeEnum
+
+
+class TextWidget(str, Enum):
+    text = "text"  # Simple text widget input, default
+    textarea = "textarea"  # A paragraph editor
+
+
+class TextInputUI(BaseModel):
+    model_config = PydanticConfigDict(extra="forbid")
+
+    widget: TextWidget = Field(
+        default=TextWidget.text,
+        description="What kind of text ui to present",
+    )
+
+
+class FileUploadUI(BaseModel):
+    model_config = PydanticConfigDict(extra="forbid")
+    accept: str = Field(
+        default="*/*",
+        description="The mime type(s) to accept in the file upload.",
+    )
+
+
+UIType = Union[TextInputUI, FileUploadUI]
+
+UI_INPUT_TO_TYPE = {
+    (TextInputUI, PrimitiveTypeEnum.text),
+    (FileUploadUI, PrimitiveTypeEnum.file),
+}

qtype/commands/convert.py

@@ -23,6 +23,9 @@ def _convert_to_yaml(doc: Application | ToolList) -> str:
     else:
         wrapped = doc
 
+    import pprint
+
+    pprint.pprint(wrapped)
     # NOTE: We use exclude_none but NOT exclude_unset because discriminator
     # fields like 'type' have default values and must be included in output
     return to_yaml_str(wrapped, exclude_none=True)

qtype/dsl/model.py

@@ -12,6 +12,7 @@ from pydantic import (
     Field,
     RootModel,
     ValidationInfo,
+    model_serializer,
     model_validator,
 )
 
@@ -23,15 +24,19 @@ from qtype.base.types import (
     ConcurrentStepMixin,
     PrimitiveTypeEnum,
     Reference,
-    StepCardinality,
     StrictBaseModel,
 )
+from qtype.base.ui_shapes import UI_INPUT_TO_TYPE, UIType
 from qtype.dsl.domain_types import (
+    AggregateStats,
     ChatContent,
     ChatMessage,
     Embedding,
+    MessageRole,
     RAGChunk,
     RAGDocument,
+    RAGSearchResult,
+    SearchResult,
 )
 
 DOMAIN_CLASSES = {
@@ -212,12 +217,24 @@ class Variable(StrictBaseModel):
         ),
     )
 
+    ui: UIType | None = Field(None, description="Hints for the UI if needed.")
+
     @model_validator(mode="before")
     @classmethod
     def resolve_type(cls, data: Any, info: ValidationInfo) -> Any:
         """Resolve string-based type references using the shared validator."""
         return _resolve_type_field_validator(data, info)
 
+    @model_validator(mode="after")
+    def validate_ui_type(self) -> Variable:
+        """Ensure at least one credential source is provided."""
+        if self.ui is not None:
+            if (type(self.ui), self.type) not in UI_INPUT_TO_TYPE:
+                raise ValueError(
+                    f"Variable of {self.type} is not comptabile with UI configuration {self.ui}"
+                )
+        return self
+
 
 class SecretReference(StrictBaseModel):
     """
@@ -257,6 +274,21 @@ class ToolParameter(BaseModel):
         """Resolve string-based type references using the shared validator."""
         return _resolve_type_field_validator(data, info)
 
+    @staticmethod
+    def _serialize_type(value):
+        if isinstance(value, type):
+            return value.__name__
+        elif hasattr(value, "__name__"):
+            return value.__name__
+        return value
+
+    @model_serializer
+    def _model_serializer(self):
+        # Use the default serialization, but ensure 'type' is a string
+        data = self.model_dump()
+        data["type"] = self._serialize_type(data.get("type"))
+        return data
+
 
 class ListType(BaseModel):
     """Represents a list type with a specific element type."""
@@ -276,12 +308,16 @@ class ListType(BaseModel):
 
 VariableType = (
     PrimitiveTypeEnum
-    | Type[Embedding]
-    | Type[ChatMessage]
-    | Type[ChatContent]
+    | Type[AggregateStats]
     | Type[BaseModel]
-    | Type[RAGDocument]
+    | Type[ChatContent]
+    | Type[ChatMessage]
+    | Type[Embedding]
+    | Type[MessageRole]
     | Type[RAGChunk]
+    | Type[RAGDocument]
+    | Type[RAGSearchResult]
+    | Type[SearchResult]
     | ListType
 )
 
@@ -350,10 +386,6 @@ class Step(CachedStepMixin, StrictBaseModel, ABC):
 
     id: str = Field(..., description="Unique ID of this component.")
     type: str = Field(..., description="Type of the step component.")
-    cardinality: StepCardinality = Field(
-        default=StepCardinality.one,
-        description="Does this step emit 1 (one) or 0...N (many) instances of the outputs?",
-    )
     inputs: list[Reference[Variable] | str] = Field(
         default_factory=list,
         description="References to the variables required by this step.",
@@ -364,6 +396,33 @@ class Step(CachedStepMixin, StrictBaseModel, ABC):
     )
 
 
+class Explode(Step):
+    """A step that takes a list input and produces multiple outputs, one per item in the list."""
+
+    type: Literal["Explode"] = "Explode"
+
+
+class Collect(Step, BatchableStepMixin):
+    """A step that collects all inputs and creates a single list to return."""
+
+    type: Literal["Collect"] = "Collect"
+
+    batch_config: BatchConfig = Field(
+        default_factory=partial(BatchConfig, batch_size=sys.maxsize),
+        description="Configuration for processing the input stream in batches. If omitted, the step processes items one by one.",
+    )
+
+
+class Construct(Step):
+    """A step that converts variables into an instance of a Custom or Domain Type"""
+
+    type: Literal["Construct"] = "Construct"
+    field_mapping: dict[str, str] = Field(
+        ...,
+        description="Mapping of type inputs to variable names, if needed.",
+    )
+
+
 class PromptTemplate(Step):
     """Defines a prompt template with a string format and variable bindings.
     This is used to generate prompts dynamically based on input variables."""
@@ -892,10 +951,6 @@ class Source(Step):
     """Base class for data sources"""
 
     id: str = Field(..., description="Unique ID of the data source.")
-    cardinality: Literal[StepCardinality.many] = Field(
-        default=StepCardinality.many,
-        description="Sources always emit 0...N instances of the outputs.",
-    )
 
 
 class SQLSource(Source):
@@ -952,7 +1007,6 @@ class Aggregate(Step):
     """
 
     type: Literal["Aggregate"] = "Aggregate"
-    cardinality: Literal[StepCardinality.one] = StepCardinality.one
 
     # Outputs are now optional. The user can provide 0, 1, 2, or 3 names.
     # The order will be: success_count, error_count, total_count
@@ -1006,10 +1060,6 @@ class DocumentSplitter(Step, ConcurrentStepMixin):
     """Configuration for chunking/splitting documents into embeddable nodes/chunks."""
 
     type: Literal["DocumentSplitter"] = "DocumentSplitter"
-    cardinality: Literal[StepCardinality.many] = Field(
-        default=StepCardinality.many,
-        description="Consumes one document and emits 0...N nodes/chunks.",
-    )
 
     splitter_name: str = Field(
         default="SentenceSplitter",
@@ -1029,10 +1079,6 @@ class DocumentEmbedder(Step, ConcurrentStepMixin):
     """Embeds document chunks using a specified embedding model."""
 
     type: Literal["DocumentEmbedder"] = "DocumentEmbedder"
-    cardinality: Literal[StepCardinality.many] = Field(
-        default=StepCardinality.many,
-        description="Consumes one chunk and emits one embedded chunk.",
-    )
     model: Reference[EmbeddingModel] | str = Field(
         ..., description="Embedding model to use for vectorization."
     )
@@ -1200,6 +1246,8 @@ StepType = Annotated[
         Agent,
         Aggregate,
         BedrockReranker,
+        Collect,
+        Construct,
         Decoder,
         DocToTextConverter,
         DocumentEmbedder,
@@ -1207,6 +1255,7 @@ StepType = Annotated[
         DocumentSplitter,
         DocumentSource,
         Echo,
+        Explode,
         FieldExtractor,
         FileSource,
         FileWriter,

qtype/interpreter/base/base_step_executor.py

@@ -230,21 +230,12 @@ class StepExecutor(ABC):
                     prepared_messages, process_item, task_limit=num_workers
                 )
 
-                # Combine all streams
-                async def emit_failed_messages() -> AsyncIterator[FlowMessage]:
-                    for msg in failed_messages:
-                        yield msg
-
-                all_results = stream.concat(
-                    stream.iterate([result_stream, emit_failed_messages()])
-                )
-
                 # Track message counts for telemetry
                 message_count = 0
                 error_count = 0
 
                 # Stream results and track progress
-                async with all_results.stream() as streamer:
+                async with result_stream.stream() as streamer:
                     result: FlowMessage
                     async for result in streamer:
                         message_count += 1
@@ -255,6 +246,15 @@ class StepExecutor(ABC):
                         )
                         yield result
 
+                # Emit failed messages after processing completes
+                for msg in failed_messages:
+                    message_count += 1
+                    error_count += 1
+                    self.progress.update_for_message(
+                        msg, self.context.on_progress
+                    )
+                    yield msg
+
                 # Finalize and track those messages too
                 async for msg in self.finalize():
                     message_count += 1

qtype/interpreter/base/factory.py

@@ -2,6 +2,8 @@ from qtype.semantic.model import (
     Agent,
     Aggregate,
     BedrockReranker,
+    Collect,
+    Construct,
     Decoder,
     DocToTextConverter,
     DocumentEmbedder,
@@ -9,6 +11,7 @@ from qtype.semantic.model import (
     DocumentSource,
     DocumentSplitter,
     Echo,
+    Explode,
     FieldExtractor,
     FileSource,
     FileWriter,
@@ -32,6 +35,8 @@ EXECUTOR_REGISTRY = {
     Agent: "qtype.interpreter.executors.agent_executor.AgentExecutor",
     Aggregate: "qtype.interpreter.executors.aggregate_executor.AggregateExecutor",
     BedrockReranker: "qtype.interpreter.executors.bedrock_reranker_executor.BedrockRerankerExecutor",
+    Collect: "qtype.interpreter.executors.collect_executor.CollectExecutor",
+    Construct: "qtype.interpreter.executors.construct_executor.ConstructExecutor",
     Decoder: "qtype.interpreter.executors.decoder_executor.DecoderExecutor",
     DocToTextConverter: "qtype.interpreter.executors.doc_to_text_executor.DocToTextConverterExecutor",
     DocumentEmbedder: "qtype.interpreter.executors.document_embedder_executor.DocumentEmbedderExecutor",
@@ -39,6 +44,7 @@ EXECUTOR_REGISTRY = {
     DocumentSource: "qtype.interpreter.executors.document_source_executor.DocumentSourceExecutor",
     DocumentSplitter: "qtype.interpreter.executors.document_splitter_executor.DocumentSplitterExecutor",
     Echo: "qtype.interpreter.executors.echo_executor.EchoExecutor",
+    Explode: "qtype.interpreter.executors.explode_executor.ExplodeExecutor",
     FieldExtractor: "qtype.interpreter.executors.field_extractor_executor.FieldExtractorExecutor",
     FileSource: "qtype.interpreter.executors.file_source_executor.FileSourceExecutor",
     FileWriter: "qtype.interpreter.executors.file_writer_executor.FileWriterExecutor",

qtype/interpreter/base/step_cache.py

@@ -67,8 +67,8 @@ def from_cache_value(
 ) -> FlowMessage:
     """Reconstructs a FlowMessage from cached output values."""
     if "FlowMessage.__error__" in cache_value:
-        msg = message.model_copy(deep=True)
-        msg.error = cache_value["FlowMessage.__error__"]
-        return msg
+        return message.model_copy(
+            deep=True, update={"error": cache_value["FlowMessage.__error__"]}
+        )
     else:
         return message.copy_with_variables(cache_value)

qtype/interpreter/conversions.py

@@ -18,7 +18,8 @@ from llama_index.core.base.llms.types import (
 from llama_index.core.memory import Memory as LlamaMemory
 from llama_index.core.schema import Document as LlamaDocument
 from llama_index.core.vector_stores.types import BasePydanticVectorStore
-from opensearchpy import AsyncOpenSearch, AWSV4SignerAuth
+from opensearchpy import AsyncHttpConnection, AsyncOpenSearch
+from opensearchpy.helpers.asyncsigner import AWSV4SignerAsyncAuth
 
 from qtype.base.types import PrimitiveTypeEnum
 from qtype.dsl.domain_types import (
@@ -369,7 +370,7 @@ def to_opensearch_client(
         InterpreterError: If authentication fails or configuration is invalid
     """
     client_kwargs: dict[str, Any] = {
-        "hosts": [index.endpoint],
+        "hosts": index.endpoint,
         **index.args,
     }
 
@@ -390,15 +391,17 @@ def to_opensearch_client(
                         f"Failed to obtain AWS credentials for DocumentIndex '{index.id}'"
                     )
 
-                # Use opensearch-py's built-in AWS auth
-                aws_auth = AWSV4SignerAuth(
+                # Use opensearch-py's async AWS auth
+                aws_auth = AWSV4SignerAsyncAuth(
                     credentials,
                     auth_session.region_name or "us-east-1",  # type: ignore
+                    "aoss",  # service name for OpenSearch Serverless
                 )
 
                 client_kwargs["http_auth"] = aws_auth
                 client_kwargs["use_ssl"] = True
                 client_kwargs["verify_certs"] = True
+                client_kwargs["connection_class"] = AsyncHttpConnection
         else:
             raise InterpreterError(
                 f"Unsupported authentication type for DocumentIndex: {type(index.auth)}"

qtype/interpreter/executors/agent_executor.py

@@ -93,8 +93,7 @@ class AgentExecutor(StepExecutor, ToolExecutionMixin, FunctionToolHelper):
             logger.error(f"Agent execution failed: {e}", exc_info=True)
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)
 
     async def _process_chat(
         self,

qtype/interpreter/executors/bedrock_reranker_executor.py

@@ -160,8 +160,7 @@ class BedrockRerankerExecutor(StepExecutor):
             logger.error(f"Reranking failed: {e}", exc_info=True)
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)
 
     def _query(self, message: FlowMessage) -> str:
         """Extract the query string from the FlowMessage.

qtype/interpreter/executors/collect_executor.py

@@ -0,0 +1,76 @@
+from typing import Any, AsyncIterator
+
+from qtype.interpreter.base.batch_step_executor import BatchedStepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import Collect
+
+
+def _find_common_ancestors(messages: list[FlowMessage]) -> dict[str, Any]:
+    if not messages:
+        return {}
+
+    # 1. Start with all variables from the first message
+    common_vars = messages[0].variables.copy()
+
+    for msg in messages[1:]:
+        # 2. Identify keys that either don't exist in this message
+        #    OR have a different value (diverged)
+        diverged_keys = [
+            k
+            for k, v in common_vars.items()
+            if k not in msg.variables or msg.variables[k] != v
+        ]
+        # 3. Remove diverged keys to leave only the "Common Ancestors"
+        for k in diverged_keys:
+            common_vars.pop(k)
+
+    return common_vars
+
+
+class CollectExecutor(BatchedStepExecutor):
+    """Executor for Collect steps."""
+
+    def __init__(
+        self,
+        step: Collect,
+        context: ExecutorContext,
+        **dependencies,
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, Collect):
+            raise ValueError("CollectExecutor can only execute Collect steps.")
+        self.step = step
+
+    async def process_batch(
+        self,
+        batch: list[FlowMessage],
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a batch of FlowMessages for the Collect step.
+
+        Args:
+            batch: A list of FlowMessages to process.
+
+        Yields:
+            FlowMessages with the results of processing.
+        """
+
+        # Note that the batch processor accumulates the messages that we need,
+        # so this function isn't called until collection is ready.
+
+        # outputs[0] and inputs[0] is safe here since semantic validation ensures only one output
+        output_name = self.step.outputs[0].id
+        input_name = self.step.inputs[0].id
+
+        if len(batch) == 0:
+            # No messages to process -- yield nothing
+            return
+
+        results = []
+        for msg in batch:
+            results.append(msg.variables[input_name])
+
+        # Only variables common to all input messages are propagated at the end
+        common_ancestors = _find_common_ancestors(batch)
+        new_variables = {output_name: results} | common_ancestors
+        yield FlowMessage(session=batch[0].session, variables=new_variables)

qtype/interpreter/executors/construct_executor.py

@@ -0,0 +1,68 @@
+from typing import AsyncIterator
+
+from qtype.dsl.model import ListType
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.interpreter.typing import instantiate_variable
+from qtype.semantic.model import Construct
+
+
+class ConstructExecutor(StepExecutor):
+    """Executor for Construct steps."""
+
+    def __init__(
+        self,
+        step: Construct,
+        context: ExecutorContext,
+        **dependencies,
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, Construct):
+            raise ValueError(
+                "ConstructExecutor can only execute Construct steps."
+            )
+        self.step = step
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a FlowMessage for the Construct step.
+
+        Args:
+            message: The FlowMessage to process.
+        Yields:
+            FlowMessages with the results of processing.
+        """
+        try:
+            # Safe since semantic validation ensures exactly one output variable
+            output_var = self.step.outputs[0]
+
+            if (
+                isinstance(output_var.type, ListType)
+                or len(self.step.inputs) == 1
+            ):
+                inputs = message.variables[self.step.inputs[0].id]
+            elif hasattr(output_var.type, "model_validate"):
+                # This is a custom type (Pydantic model)
+                # So input should be a dict
+                input_values = {
+                    input_var.id: message.variables[input_var.id]
+                    for input_var in self.step.inputs
+                }
+                # use the mapping to convert variable names to
+                inputs = {
+                    self.step.field_mapping.get(var_name, var_name): value
+                    for var_name, value in input_values.items()
+                }
+            else:
+                raise ValueError(
+                    "Construct step must have either a single input or output of a custom type."
+                )
+            constructed_value = instantiate_variable(output_var, inputs)
+            yield message.copy_with_variables(
+                {output_var.id: constructed_value}
+            )
+        except Exception as e:
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/decoder_executor.py

@@ -159,5 +159,4 @@ class DecoderExecutor(StepExecutor):
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/doc_to_text_executor.py

@@ -74,8 +74,7 @@ class DocToTextConverterExecutor(StepExecutor):
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)
 
     def _convert_doc(self, doc: RAGDocument) -> RAGDocument:
         """Convert a RAGDocument to text/markdown format.

qtype/interpreter/executors/document_embedder_executor.py

@@ -119,5 +119,4 @@ class DocumentEmbedderExecutor(StepExecutor):
                 f"Error processing DocumentEmbedder step {self.step.id}",
                 exc_info=e,
             )
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/document_search_executor.py

@@ -109,5 +109,4 @@ class DocumentSearchExecutor(StepExecutor):
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/document_source_executor.py

@@ -114,5 +114,4 @@ class DocumentSourceExecutor(StepExecutor):
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/document_splitter_executor.py

@@ -96,10 +96,12 @@ class DocumentSplitterExecutor(StepExecutor):
                     vector=None,  # Embedding will be added later
                     metadata=merged_metadata,
                 )
-                yield message.copy_with_variables({output_id: chunk})
+                if (
+                    chunk.content and chunk.content.strip()
+                ):  # Only emit non-empty chunks
+                    yield message.copy_with_variables({output_id: chunk})
 
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)

qtype/interpreter/executors/echo_executor.py

@@ -59,5 +59,4 @@ class EchoExecutor(StepExecutor):
         except Exception as e:
             # Emit error event to stream so frontend can display it
             await self.stream_emitter.error(str(e))
-            message.set_error(self.step.id, e)
-            yield message
+            yield message.copy_with_error(self.step.id, e)