qtype 0.0.12__py3-none-any.whl → 0.1.3__py3-none-any.whl

qtype/interpreter/executors/document_source_executor.py

@@ -0,0 +1,118 @@
+import importlib
+from typing import AsyncIterator
+
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.conversions import from_llama_document
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import DocumentSource
+
+
+class DocumentSourceExecutor(StepExecutor):
+    """Executor for DocumentSource steps."""
+
+    def __init__(
+        self, step: DocumentSource, context: ExecutorContext, **dependencies
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, DocumentSource):
+            raise ValueError(
+                (
+                    "DocumentSourceExecutor can only execute "
+                    "DocumentSource steps."
+                )
+            )
+        self.step: DocumentSource = step
+        # Initialize the reader class once for the executor
+        self.reader_class = self._load_reader_class()
+
+    def _load_reader_class(self) -> type:
+        """Load the LlamaIndex reader class dynamically.
+
+        Returns:
+            The reader class.
+
+        Raises:
+            ImportError: If the reader class cannot be imported.
+        """
+        # Parse the reader module path
+        # Format: 'file.SimpleDirectoryReader' -> llama_index.readers.file + SimpleDirectoryReader
+        # Special case: 'file.SimpleDirectoryReader' is actually in llama_index.core
+        parts = self.step.reader_module.split(".")
+        module_path = ".".join(parts[:-1])
+        class_name = parts[-1]
+
+        # Dynamically import the reader module and get the class
+        try:
+            module = importlib.import_module(module_path)
+            reader_class = getattr(module, class_name)
+            return reader_class
+        except (ImportError, AttributeError) as e:
+            raise ImportError(
+                (
+                    f"Failed to import reader class '{class_name}' "
+                    f"from '{module_path}': {e}"
+                )
+            ) from e
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a single FlowMessage for the DocumentSource step.
+
+        Args:
+            message: The FlowMessage to process.
+        Yields:
+            FlowMessages with loaded documents.
+        """
+        output_id = self.step.outputs[0].id
+
+        try:
+            # Resolve any SecretReferences in step args
+            context = f"step '{self.step.id}'"
+            resolved_args = self._secret_manager.resolve_secrets_in_dict(
+                self.step.args, context
+            )
+
+            # Combine resolved step args with message variables as runtime args
+            runtime_args = {
+                key: message.variables.get(key)
+                for key in message.variables.keys()
+            }
+            combined_args = {**resolved_args, **runtime_args}
+
+            # Instantiate the reader with combined arguments
+            loader = self.reader_class(**combined_args)
+
+            # Load documents using the loader
+            if not hasattr(loader, "load_data"):
+                raise AttributeError(
+                    (
+                        f"Reader class '{self.reader_class.__name__}' "
+                        "does not have a 'load_data' method"
+                    )
+                )
+            load_args = self.step.loader_args or {}
+
+            llama_documents = loader.load_data(**load_args)
+
+            # Convert LlamaIndex Documents to RAGDocuments
+            rag_documents = [
+                from_llama_document(doc) for doc in llama_documents
+            ]
+
+            # Emit feedback about total documents loaded
+            await self.stream_emitter.status(
+                f"Loaded {len(rag_documents)} documents"
+            )
+
+            # Yield one message per document (fan-out)
+            for doc in rag_documents:
+                yield message.copy_with_variables({output_id: doc})
+
+        except Exception as e:
+            # Emit error event to stream so frontend can display it
+            await self.stream_emitter.error(str(e))
+            message.set_error(self.step.id, e)
+            yield message

qtype/interpreter/executors/document_splitter_executor.py

@@ -0,0 +1,105 @@
+from typing import AsyncIterator
+
+from llama_index.core.schema import Document as LlamaDocument
+
+from qtype.dsl.domain_types import RAGChunk, RAGDocument
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.conversions import to_text_splitter
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import DocumentSplitter
+
+
+class DocumentSplitterExecutor(StepExecutor):
+    """Executor for DocumentSplitter steps."""
+
+    def __init__(
+        self, step: DocumentSplitter, context: ExecutorContext, **dependencies
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, DocumentSplitter):
+            raise ValueError(
+                (
+                    "DocumentSplitterExecutor can only execute "
+                    "DocumentSplitter steps."
+                )
+            )
+        self.step: DocumentSplitter = step
+        # Initialize the text splitter once for the executor
+        self.llama_splitter = to_text_splitter(self.step)
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a single FlowMessage for the DocumentSplitter step.
+
+        Args:
+            message: The FlowMessage to process.
+
+        Yields:
+            FlowMessages with document chunks.
+        """
+        input_id = self.step.inputs[0].id
+        output_id = self.step.outputs[0].id
+
+        try:
+            # Get the document from the input
+            document = message.variables.get(input_id)
+            if not isinstance(document, RAGDocument):
+                raise ValueError(
+                    f"Input variable '{input_id}' must be a RAGDocument"
+                )
+
+            await self.stream_emitter.status(
+                f"Splitting document: {document.file_name}"
+            )
+
+            # Convert content to text if needed
+            if isinstance(document.content, bytes):
+                content_text = document.content.decode("utf-8")
+            elif isinstance(document.content, str):
+                content_text = document.content
+            else:
+                raise ValueError(
+                    (
+                        f"Unsupported document content type: "
+                        f"{type(document.content)}"
+                    )
+                )
+
+            # Convert to LlamaIndex Document for splitting
+            llama_doc = LlamaDocument(
+                text=content_text,
+                metadata={},  # ommit metadata here as it's added back later and we don't want the chunk size checks
+                doc_id=document.file_id,
+            )
+
+            # Split the document using the LlamaIndex splitter
+            nodes = self.llama_splitter.get_nodes_from_documents([llama_doc])
+
+            await self.stream_emitter.status(
+                f"Split {document.file_name} into {len(nodes)} chunks"
+            )
+
+            # Create a RAGChunk for each node and yield (fan-out)
+            for node in nodes:
+                merged_metadata = {}
+                merged_metadata.update(document.metadata)
+                if node.metadata:
+                    merged_metadata.update(node.metadata)
+
+                chunk = RAGChunk(
+                    content=node.text,
+                    chunk_id=node.node_id,
+                    document_id=document.file_id,
+                    vector=None,  # Embedding will be added later
+                    metadata=merged_metadata,
+                )
+                yield message.copy_with_variables({output_id: chunk})
+
+        except Exception as e:
+            # Emit error event to stream so frontend can display it
+            await self.stream_emitter.error(str(e))
+            message.set_error(self.step.id, e)
+            yield message

qtype/interpreter/executors/echo_executor.py

@@ -0,0 +1,63 @@
+from typing import AsyncIterator
+
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import Echo
+
+
+class EchoExecutor(StepExecutor):
+    """Executor for Echo steps.
+
+    Passes through input variables as outputs without modification.
+    Useful for debugging flows by inspecting variable values at specific
+    points in the execution pipeline.
+    """
+
+    def __init__(
+        self,
+        step: Echo,
+        context: ExecutorContext,
+        **dependencies: object,
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, Echo):
+            raise ValueError("EchoExecutor can only execute Echo steps.")
+        self.step: Echo = step
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a single FlowMessage for the Echo step.
+
+        Reads all input variables from the message and copies them to
+        the output variables with the same IDs.
+
+        Args:
+            message: The FlowMessage to process.
+
+        Yields:
+            FlowMessage with the echoed variables.
+        """
+        try:
+            # Build a dict of output variable values by reading from inputs
+            output_vars = {}
+            for input_var in self.step.inputs:
+                value = message.variables.get(input_var.id)
+                # Find the corresponding output variable ID (should match)
+                for output_var in self.step.outputs:
+                    if output_var.id == input_var.id:
+                        output_vars[output_var.id] = value
+                        break
+
+            await self.stream_emitter.status(
+                f"Echoed {len(output_vars)} variable(s) in step {self.step.id}",
+            )
+            yield message.copy_with_variables(output_vars)
+
+        except Exception as e:
+            # Emit error event to stream so frontend can display it
+            await self.stream_emitter.error(str(e))
+            message.set_error(self.step.id, e)
+            yield message

qtype/interpreter/executors/field_extractor_executor.py

@@ -0,0 +1,165 @@
+from typing import Any, AsyncIterator
+
+from jsonpath_ng.ext import parse  # type: ignore[import-untyped]
+from pydantic import BaseModel
+
+from qtype.base.types import PrimitiveTypeEnum
+from qtype.dsl.model import ListType
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import FieldExtractor
+
+
+class FieldExtractorExecutor(StepExecutor):
+    """Executor for FieldExtractor steps.
+
+    Extracts fields from input data using JSONPath expressions and
+    constructs output instances. Supports 1-to-many cardinality when
+    the JSONPath matches multiple values.
+    """
+
+    def __init__(
+        self,
+        step: FieldExtractor,
+        context: ExecutorContext,
+        **dependencies: object,
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, FieldExtractor):
+            raise ValueError(
+                "FieldExtractorExecutor can only execute FieldExtractor steps."
+            )
+        self.step: FieldExtractor = step
+
+        # Parse the JSONPath expression once at initialization
+        try:
+            self.jsonpath_expr = parse(self.step.json_path)
+        except Exception as e:
+            raise ValueError(
+                f"Invalid JSONPath expression '{self.step.json_path}': {e}"
+            ) from e
+
+    def _to_dict(self, value: Any) -> Any:
+        """Convert value to dict representation for JSONPath processing.
+
+        Args:
+            value: The value to convert (could be BaseModel, dict, list, etc.)
+
+        Returns:
+            Dict representation suitable for JSONPath processing
+        """
+        if isinstance(value, BaseModel):
+            return value.model_dump()
+        return value
+
+    def _construct_output(self, extracted_data: Any) -> Any:
+        """Construct the output value from extracted data.
+
+        Args:
+            extracted_data: The data extracted by JSONPath
+
+        Returns:
+            Constructed output value based on the output variable type
+        """
+        output_var = self.step.outputs[0]
+        output_type = output_var.type
+
+        # Handle primitive types - just return the extracted data
+        if isinstance(output_type, PrimitiveTypeEnum):
+            return extracted_data
+
+        # Handle list types
+        if isinstance(output_type, ListType):
+            # The extracted_data should already be a list
+            if not isinstance(extracted_data, list):
+                extracted_data = [extracted_data]
+            return extracted_data
+
+        # Handle BaseModel types (domain types and custom types)
+        if isinstance(output_type, type) and issubclass(
+            output_type, BaseModel
+        ):
+            # If extracted_data is a dict, use it as kwargs
+            if isinstance(extracted_data, dict):
+                return output_type(**extracted_data)
+            # If it's already the right type, return it
+            elif isinstance(extracted_data, output_type):
+                return extracted_data
+            else:
+                raise ValueError(
+                    (
+                        f"Cannot construct {output_type.__name__} from "
+                        f"{type(extracted_data).__name__}"
+                    )
+                )
+
+        # Fallback - return as-is
+        return extracted_data
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a single FlowMessage for the FieldExtractor step.
+
+        Args:
+            message: The FlowMessage to process.
+
+        Yields:
+            FlowMessage(s) with extracted and constructed output values.
+            Multiple messages may be yielded if JSONPath matches multiple values.
+        """
+        input_id = self.step.inputs[0].id
+        output_id = self.step.outputs[0].id
+
+        try:
+            # Get the input value
+            input_value = message.variables.get(input_id)
+            if input_value is None:
+                raise ValueError(
+                    f"Input variable '{input_id}' is not set or is None"
+                )
+
+            await self.stream_emitter.status(
+                f"Extracting fields using JSONPath: {self.step.json_path}"
+            )
+
+            # Convert input to dict for JSONPath processing
+            input_dict = self._to_dict(input_value)
+
+            # Apply JSONPath expression
+            matches = self.jsonpath_expr.find(input_dict)
+
+            if not matches:
+                if self.step.fail_on_missing:
+                    raise ValueError(
+                        (
+                            f"JSONPath expression '{self.step.json_path}' "
+                            f"did not match any data in input"
+                        )
+                    )
+                else:
+                    # Yield message with None output
+                    yield message.copy_with_variables({output_id: None})
+                    return
+
+            await self.stream_emitter.status(
+                f"JSONPath matched {len(matches)} value(s)"
+            )
+
+            # Yield one message per match (1-to-many)
+            for match in matches:
+                extracted_data = match.value
+
+                # Construct the output value
+                output_value = self._construct_output(extracted_data)
+
+                # Yield message with the constructed output
+                yield message.copy_with_variables({output_id: output_value})
+
+        except Exception as e:
+            # Emit error event to stream so frontend can display it
+            await self.stream_emitter.error(str(e))
+            message.set_error(self.step.id, e)
+            yield message

qtype/interpreter/executors/file_source_executor.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import AsyncIterator
+
+import fsspec
+import pandas as pd
+
+from qtype.interpreter.base.base_step_executor import StepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import ConstantPath, FileSource
+
+
+class FileSourceExecutor(StepExecutor):
+    """Executor for FileSource steps."""
+
+    def __init__(
+        self, step: FileSource, context: ExecutorContext, **dependencies
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, FileSource):
+            raise ValueError(
+                "FileSourceExecutor can only execute FileSource steps."
+            )
+        self.step = step
+
+    async def process_message(
+        self,
+        message: FlowMessage,
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a single FlowMessage for the FileSource step.
+
+        Args:
+            message: The FlowMessage to process.
+
+        Yields:
+            FlowMessages with the results of processing.
+        """
+        output_columns = {output.id for output in self.step.outputs}
+
+        # get the path
+        if isinstance(self.step.path, ConstantPath):
+            file_path = self.step.path
+        else:
+            file_path = message.variables.get(self.step.path.id)
+            if not file_path:
+                raise ValueError(
+                    (
+                        f"FileSource step {self.step.id} requires a path "
+                        "variable."
+                    )
+                )
+            await self.stream_emitter.status(
+                f"Reading file from path: {file_path}"
+            )
+
+        # Determine file format from extension
+        file_path_str = (
+            file_path.uri if isinstance(file_path, ConstantPath) else file_path
+        )
+        extension = Path(file_path_str).suffix.lower()
+
+        # Use fsspec to open the file and read with pandas
+        with fsspec.open(file_path_str, "rb") as file_handle:
+            if extension == ".csv":
+                df = pd.read_csv(file_handle)  # type: ignore[arg-type]
+            elif extension == ".parquet":
+                df = pd.read_parquet(file_handle)  # type: ignore[arg-type]
+            elif extension == ".json":
+                df = pd.read_json(file_handle)  # type: ignore[arg-type]
+            elif extension == ".jsonl":
+                df = pd.read_json(
+                    file_handle,
+                    lines=True,  # type: ignore[arg-type]
+                )
+            else:
+                # Default to parquet if no extension or unknown
+                df = pd.read_parquet(file_handle)  # type: ignore[arg-type]
+
+        # confirm the outputs exist in the dataframe
+        columns = set(df.columns)
+        missing_columns = output_columns - columns
+        if missing_columns:
+            raise ValueError(
+                (
+                    f"File {file_path_str} missing expected columns: "
+                    f"{', '.join(missing_columns)}. Available columns: "
+                    f"{', '.join(columns)}"
+                )
+            )
+
+        for row in df.to_dict(orient="records"):
+            # Filter to only the expected output columns if they exist
+            row = {
+                str(k): v for k, v in row.items() if str(k) in output_columns
+            }
+            yield message.copy_with_variables(new_variables=row)
+        await self.stream_emitter.status(
+            f"Emitted {len(df)} rows from: {file_path_str}"
+        )

qtype/interpreter/executors/file_writer_executor.py

@@ -0,0 +1,110 @@
+from itertools import groupby
+from typing import AsyncIterator
+
+import fsspec
+import pandas as pd
+
+from qtype.interpreter.base.batch_step_executor import BatchedStepExecutor
+from qtype.interpreter.base.executor_context import ExecutorContext
+from qtype.interpreter.types import FlowMessage
+from qtype.semantic.model import ConstantPath, FileWriter, Variable
+
+
+class FileWriterExecutor(BatchedStepExecutor):
+    """Executor for FileWriter steps."""
+
+    def __init__(
+        self,
+        step: FileWriter,
+        context: ExecutorContext,
+        **dependencies,
+    ):
+        super().__init__(step, context, **dependencies)
+        if not isinstance(step, FileWriter):
+            raise ValueError(
+                "FileWriterExecutor can only execute FileWriter steps."
+            )
+        self.step = step
+
+    def to_pandas(self, messages: list[FlowMessage]) -> pd.DataFrame:
+        """Convert a list of FlowMessages to a pandas DataFrame."""
+        records = [msg.variables for msg in messages]
+        return pd.DataFrame.from_records(records)
+
+    async def process_batch(
+        self,
+        batch: list[FlowMessage],
+    ) -> AsyncIterator[FlowMessage]:
+        """Process a batch of FlowMessages for the FileWriter step.
+
+        Args:
+            batch: A list of FlowMessages to process.
+
+        Yields:
+            FlowMessages with the results of processing.
+        """
+        output_name = None
+        if len(self.step.outputs):
+            output_name = self.step.outputs[0].id
+
+        if isinstance(self.step.path, ConstantPath):
+            file_path = self.step.path.uri
+            df = self.to_pandas(batch)
+            # A fixed path is provided -- just write all of the data
+            await self.stream_emitter.status(
+                f"Writing {len(df)} records to {file_path}"
+            )
+            with fsspec.open(file_path, "wb") as file_handle:
+                df.to_parquet(file_handle, index=False)  # type: ignore[arg-type]
+            await self.stream_emitter.status(
+                f"Wrote {len(df)} records to {file_path}"
+            )
+            for msg in batch:
+                yield (
+                    msg
+                    if not output_name
+                    else msg.copy_with_variables({output_name: file_path})
+                )
+        else:
+            # Group messages by file path (path is a Variable in this branch)
+            if not isinstance(self.step.path, Variable):
+                raise ValueError(
+                    "Expected path to be a Variable in dynamic path case."
+                )
+
+            path_var_id = self.step.path.id
+
+            # Sort messages by file path for groupby
+            sorted_batch = sorted(
+                batch, key=lambda msg: msg.variables[path_var_id]
+            )
+
+            # Group messages by file path
+            grouped_messages = groupby(
+                sorted_batch, key=lambda msg: msg.variables[path_var_id]
+            )
+
+            distinct_paths = list(
+                set(msg.variables[path_var_id] for msg in batch)
+            )
+            await self.stream_emitter.status(
+                f"There are {len(distinct_paths)} different files to write."
+            )
+            for file_path, msg_group in grouped_messages:
+                msg_list = list(msg_group)
+                df_group = self.to_pandas(msg_list)
+                await self.stream_emitter.status(
+                    f"Writing {len(df_group)} records to {file_path}"
+                )
+                with fsspec.open(file_path, "wb") as file_handle:
+                    df_group.to_parquet(file_handle, index=False)  # type: ignore[arg-type]
+                await self.stream_emitter.status(
+                    f"Wrote {len(df_group)} records to {file_path}"
+                )
+                # Re-yield the original messages for this group
+                for msg in batch:
+                    yield (
+                        msg
+                        if not output_name
+                        else msg.copy_with_variables({output_name: file_path})
+                    )