qtype 0.0.16__tar.gz → 0.1.1__tar.gz

{qtype-0.0.16/qtype.egg-info → qtype-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qtype
-Version: 0.0.16
+Version: 0.1.1
 Summary: DSL for Generative AI Prototyping
 Author-email: Lou Kratz <lou.kratz+qtype@bazaarvoice.com>
 License-Expression: Apache-2.0
@@ -15,20 +15,24 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: openai>=1.93.0
 Requires-Dist: fsspec>=2025.5.1
 Requires-Dist: mkdocs-awesome-pages-plugin>=2.10.1
-Requires-Dist: mermaid-py>=0.8.0
 Requires-Dist: pip-system-certs>=5.2
 Requires-Dist: openapi3-parser>=1.1.21
 Requires-Dist: pydantic-yaml>=1.6.0
+Requires-Dist: google-cloud-aiplatform>=1.120.0
 Provides-Extra: interpreter
+Requires-Dist: aiostream>=0.7.1; extra == "interpreter"
 Requires-Dist: arize-phoenix-otel>=0.12.1; extra == "interpreter"
 Requires-Dist: boto3>=1.34.0; extra == "interpreter"
+Requires-Dist: docling>=2.55.1; extra == "interpreter"
+Requires-Dist: diskcache[interpreter]>=5.6.3; extra == "interpreter"
 Requires-Dist: fastapi>=0.116.1; extra == "interpreter"
 Requires-Dist: llama-index-embeddings-bedrock>=0.5.2; extra == "interpreter"
 Requires-Dist: llama-index-embeddings-openai>=0.3.1; extra == "interpreter"
-Requires-Dist: llama-index-llms-bedrock-converse>=0.7.4; extra == "interpreter"
+Requires-Dist: llama-index-llms-bedrock-converse>=0.10.5; extra == "interpreter"
 Requires-Dist: llama-index-llms-bedrock>=0.3.8; extra == "interpreter"
 Requires-Dist: llama-index>=0.12.45; extra == "interpreter"
 Requires-Dist: openinference-instrumentation-llama-index>=4.3.4; extra == "interpreter"
+Requires-Dist: opensearch-py>=2.7.0; extra == "interpreter"
 Requires-Dist: pandas>=2.2.3; extra == "interpreter"
 Requires-Dist: psycopg2-binary>=2.9.10; extra == "interpreter"
 Requires-Dist: pyarrow>=21.0.0; extra == "interpreter"
@@ -37,6 +41,15 @@ Requires-Dist: python-magic>=0.4.27; extra == "interpreter"
 Requires-Dist: s3fs>=2025.7.0; extra == "interpreter"
 Requires-Dist: sqlalchemy>=2.0.42; extra == "interpreter"
 Requires-Dist: uvicorn[standard]>=0.35.0; extra == "interpreter"
+Requires-Dist: llama-index-llms-vertex>=0.6.1; extra == "interpreter"
+Requires-Dist: langfuse>=3.9.0; extra == "interpreter"
+Requires-Dist: opentelemetry-exporter-otlp>=1.35.0; extra == "interpreter"
+Requires-Dist: opentelemetry-sdk>=1.35.0; extra == "interpreter"
+Requires-Dist: docx2txt>=0.9; extra == "interpreter"
+Requires-Dist: llama-index-vector-stores-qdrant>=0.8.6; extra == "interpreter"
+Requires-Dist: jsonpath-ng>=1.7.0; extra == "interpreter"
+Requires-Dist: llama-index-readers-huggingface-fs>=0.4.1; extra == "interpreter"
+Requires-Dist: datasets>=4.4.1; extra == "interpreter"
 Dynamic: license-file
 
 # QType

{qtype-0.0.16 → qtype-0.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "qtype"
-version = "0.0.16"
+version = "0.1.1"
 description = "DSL for Generative AI Prototyping"
 authors = [{ name="Lou Kratz", email="lou.kratz+qtype@bazaarvoice.com" }]
 readme = "README.md"
@@ -13,10 +13,10 @@ dependencies = [
     "openai>=1.93.0",
     "fsspec>=2025.5.1",
     "mkdocs-awesome-pages-plugin>=2.10.1",
-    "mermaid-py>=0.8.0",
     "pip-system-certs>=5.2",
     "openapi3-parser>=1.1.21",
     "pydantic-yaml>=1.6.0",
+    "google-cloud-aiplatform>=1.120.0",
 ]
 license = "APACHE-2.0"
 license-files = ["LICEN[CS]E*"]
@@ -26,15 +26,19 @@ Homepage = "https://github.com/bazaarvoice/qtype"
 
 [project.optional-dependencies]
 interpreter = [
+    "aiostream>=0.7.1",
     "arize-phoenix-otel>=0.12.1",
     "boto3>=1.34.0",
+    "docling>=2.55.1",
+    "diskcache[interpreter]>=5.6.3",
     "fastapi>=0.116.1",
     "llama-index-embeddings-bedrock>=0.5.2",
     "llama-index-embeddings-openai>=0.3.1",
-    "llama-index-llms-bedrock-converse>=0.7.4",
+    "llama-index-llms-bedrock-converse>=0.10.5",
     "llama-index-llms-bedrock>=0.3.8",
     "llama-index>=0.12.45",
     "openinference-instrumentation-llama-index>=4.3.4",
+    "opensearch-py>=2.7.0",
     "pandas>=2.2.3",
     "psycopg2-binary>=2.9.10",
     "pyarrow>=21.0.0",
@@ -43,6 +47,15 @@ interpreter = [
     "s3fs>=2025.7.0",
     "sqlalchemy>=2.0.42",
     "uvicorn[standard]>=0.35.0",
+    "llama-index-llms-vertex>=0.6.1",
+    "langfuse>=3.9.0",
+    "opentelemetry-exporter-otlp>=1.35.0",
+    "opentelemetry-sdk>=1.35.0",
+    "docx2txt>=0.9",
+    "llama-index-vector-stores-qdrant>=0.8.6",
+    "jsonpath-ng>=1.7.0",
+    "llama-index-readers-huggingface-fs>=0.4.1",
+    "datasets>=4.4.1",
 ]
 
 [dependency-groups]
@@ -52,6 +65,7 @@ dev = [
     "coverage>=7.0.0",
     "ipython>=8.37.0",
     "isort>=5.13.0",
+    "mermaid-py>=0.8.0",
     "mkdocs-awesome-pages-plugin>=2.10.1",
     "mkdocs-material>=9.6.15",
     "mkdocs>=1.6.1",
@@ -63,6 +77,7 @@ dev = [
     "pkginfo>=1.12.1.2",
     "pre-commit>=3.6.0",
     "pymdown-extensions>=10.16",
+    "pytest-asyncio>=1.2.0",
     "pytest-cov>=6.0.0",
     "pytest>=8.4.1",
     "ruff>=0.1.0",
@@ -79,6 +94,8 @@ docs = [
 [tool.uv]
 # Install dev dependencies by default when running uv sync
 default-groups = ["dev"]
+# Install the package itself in editable mode
+package = true
 # Use highest resolution strategy for dependencies  
 resolution = "highest"
 # Compile bytecode for faster subsequent runs
@@ -183,4 +200,7 @@ filterwarnings = [
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "network: marks tests as requiring network access",
+    "asyncio: marks tests as async tests using asyncio",
 ]
+# Configure pytest-asyncio to automatically detect async tests
+asyncio_mode = "auto"

{qtype-0.0.16 → qtype-0.1.1}/qtype/application/commons/tools.py

@@ -180,7 +180,7 @@ def parse_duration_string(duration: str) -> int:
 
 def format_datetime(timestamp: datetime, format_string: str) -> str:
     """
-    Format a timestamp using a custom format string.
+    Format a timestamp using a custom format string that can be passed to strftime.
 
     Args:
         timestamp: Datetime object to format.

{qtype-0.0.16 → qtype-0.1.1}/qtype/application/converters/tools_from_api.py

@@ -17,7 +17,7 @@ from openapi_parser.specification import (
     Security,
 )
 
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 from qtype.dsl.model import (
     APIKeyAuthProvider,
     APITool,
@@ -344,9 +344,9 @@ def to_api_tool(
         endpoint=endpoint,
         method=operation.method.value.upper(),
         auth=auth.id if auth else None,  # Use auth ID string instead of object
-        inputs=inputs if inputs else None,
-        outputs=outputs if outputs else None,
-        parameters=parameters if parameters else None,
+        inputs=inputs,
+        outputs=outputs,
+        parameters=parameters,
     )
 
 
@@ -394,7 +394,7 @@ def to_authorization_provider(
                     }
                 )
                 if any(flow.scopes for flow in security.flows.values())
-                else None,
+                else [],
             )
         case _:
             raise ValueError(

{qtype-0.0.16 → qtype-0.1.1}/qtype/application/converters/tools_from_module.py

@@ -5,7 +5,7 @@ from typing import Any, Type, Union, get_args, get_origin
 from pydantic import BaseModel
 
 from qtype.application.converters.types import PYTHON_TYPE_TO_PRIMITIVE_TYPE
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 from qtype.dsl.model import (
     CustomType,
     ListType,
@@ -159,7 +159,7 @@ def _create_tool_from_function(
         module_path=func_info["module"],
         function_name=func_name,
         description=description,
-        inputs=inputs if inputs else None,
+        inputs=inputs,
         outputs=outputs,
     )
 

qtype-0.1.1/qtype/application/converters/types.py

@@ -0,0 +1,18 @@
+"""
+Re-export type mappings from DSL layer for backward compatibility.
+
+This module maintains the application layer's interface while delegating
+to the DSL layer where these mappings are now defined.
+"""
+
+from qtype.dsl.types import (
+    PRIMITIVE_TO_PYTHON_TYPE,
+    PYTHON_TYPE_TO_PRIMITIVE_TYPE,
+    python_type_for_list,
+)
+
+__all__ = [
+    "PRIMITIVE_TO_PYTHON_TYPE",
+    "PYTHON_TYPE_TO_PRIMITIVE_TYPE",
+    "python_type_for_list",
+]

{qtype-0.0.16 → qtype-0.1.1}/qtype/application/documentation.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any, Type, Union, get_args, get_origin
 
 import qtype.dsl.model as dsl
-from qtype.dsl.base_types import PrimitiveTypeEnum
+from qtype.base.types import PrimitiveTypeEnum
 
 
 def _format_type_name(field_type: Any) -> str:

qtype-0.1.1/qtype/application/facade.py

@@ -0,0 +1,178 @@
+"""Main facade for qtype operations."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from qtype.base.types import PathLike
+from qtype.semantic.model import Application as SemanticApplication
+from qtype.semantic.model import DocumentType as SemanticDocumentType
+
+# Note: There should be _zero_ imports here at the top that import qtype.interpreter.
+# That's the whole point of this facade - to avoid importing optional
+# dependencies unless these methods are called.
+
+logger = logging.getLogger(__name__)
+
+
+class QTypeFacade:
+    """
+    Simplified interface for qtype operations.
+
+    This facade provides lazy-loading wrappers for operations that require
+    optional dependencies (interpreter package), allowing base qtype to work
+    without those dependencies installed.
+    """
+
+    def telemetry(self, spec: SemanticDocumentType) -> None:
+        if isinstance(spec, SemanticApplication) and spec.telemetry:
+            logger.info(
+                f"Telemetry enabled with endpoint: {spec.telemetry.endpoint}"
+            )
+            # Register telemetry if needed
+            from qtype.interpreter.telemetry import register
+
+            register(spec.telemetry, self.secret_manager(spec), spec.id)
+
+    def secret_manager(self, spec: SemanticDocumentType):
+        """
+        Create a secret manager based on the specification.
+
+        Args:
+            spec: SemanticDocumentType specification
+
+        Returns:
+            Secret manager instance
+        """
+        from qtype.interpreter.base.secrets import create_secret_manager
+
+        if isinstance(spec, SemanticApplication):
+            return create_secret_manager(spec.secret_manager)
+        else:
+            raise ValueError(
+                "Can't create secret manager for non-Application spec"
+            )
+
+    async def execute_workflow(
+        self,
+        path: PathLike,
+        inputs: dict | Any,
+        flow_name: str | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Execute a complete workflow from document to results.
+
+        Args:
+            path: Path to the QType specification file
+            inputs: Dictionary of input values or DataFrame for batch
+            flow_name: Optional name of flow to execute
+            **kwargs: Additional dependencies for execution
+
+        Returns:
+            DataFrame with results (one row per input)
+        """
+        import pandas as pd
+
+        from qtype.semantic.loader import load
+
+        logger.info(f"Executing workflow from {path}")
+
+        # Load the semantic application
+        semantic_model, type_registry = load(Path(path))
+        assert isinstance(semantic_model, SemanticApplication)
+        self.telemetry(semantic_model)
+
+        # Find the flow to execute
+        if flow_name:
+            target_flow = None
+            for flow in semantic_model.flows:
+                if flow.id == flow_name:
+                    target_flow = flow
+                    break
+            if target_flow is None:
+                raise ValueError(f"Flow '{flow_name}' not found")
+        else:
+            if semantic_model.flows:
+                target_flow = semantic_model.flows[0]
+            else:
+                raise ValueError("No flows found in application")
+
+        # Convert inputs to DataFrame (normalize single dict to 1-row DataFrame)
+        if isinstance(inputs, dict):
+            input_df = pd.DataFrame([inputs])
+        elif isinstance(inputs, pd.DataFrame):
+            input_df = inputs
+        else:
+            raise ValueError(
+                f"Inputs must be dict or DataFrame, got {type(inputs)}"
+            )
+
+        # Create session
+        from qtype.interpreter.converters import (
+            dataframe_to_flow_messages,
+            flow_messages_to_dataframe,
+        )
+        from qtype.interpreter.types import Session
+
+        session = Session(
+            session_id=kwargs.pop("session_id", "default"),
+            conversation_history=kwargs.pop("conversation_history", []),
+        )
+
+        # Convert DataFrame to FlowMessages
+        initial_messages = dataframe_to_flow_messages(input_df, session)
+
+        # Execute the flow
+        from opentelemetry import trace
+
+        from qtype.interpreter.base.executor_context import ExecutorContext
+        from qtype.interpreter.flow import run_flow
+
+        secret_manager = self.secret_manager(semantic_model)
+        context = ExecutorContext(
+            secret_manager=secret_manager,
+            tracer=trace.get_tracer(__name__),
+        )
+        results = await run_flow(
+            target_flow,
+            initial_messages,
+            context=context,
+            **kwargs,
+        )
+
+        # Convert results back to DataFrame
+        results_df = flow_messages_to_dataframe(results, target_flow)
+
+        return results_df
+
+    def generate_aws_bedrock_models(self) -> list[dict[str, Any]]:
+        """
+        Generate AWS Bedrock model definitions.
+
+        Returns:
+            List of model definitions for AWS Bedrock models.
+
+        Raises:
+            ImportError: If boto3 is not installed.
+            Exception: If AWS API call fails.
+        """
+        import boto3  # type: ignore[import-untyped]
+
+        logger.info("Discovering AWS Bedrock models...")
+        client = boto3.client("bedrock")
+        models = client.list_foundation_models()
+
+        model_definitions = []
+        for model_summary in models.get("modelSummaries", []):
+            model_definitions.append(
+                {
+                    "id": model_summary["modelId"],
+                    "provider": "aws-bedrock",
+                }
+            )
+
+        logger.info(f"Discovered {len(model_definitions)} AWS Bedrock models")
+        return model_definitions

qtype-0.1.1/qtype/base/types.py

@@ -0,0 +1,249 @@
+"""Common type definitions for qtype."""
+
+from __future__ import annotations
+
+import pathlib
+import types
+import typing
+from enum import Enum
+from typing import (
+    Any,
+    Generic,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
+
+from pydantic import BaseModel
+from pydantic import ConfigDict as PydanticConfigDict
+from pydantic import Field, model_validator
+
+# JSON-serializable value types
+JSONValue = Union[
+    str,
+    int,
+    float,
+    bool,
+    None,
+    dict[str, "JSONValue"],
+    list["JSONValue"],
+]
+
+# Path-like type (string or Path object)
+PathLike = Union[str, pathlib.Path]
+
+CustomTypeRegistry = dict[str, Type[BaseModel]]
+# Configuration dictionary type
+ConfigDict = dict[str, Any]
+
+
+# ---------------- Shared Base Types and Enums ----------------
+
+
+class PrimitiveTypeEnum(str, Enum):
+    """Represents the type of data a user or system input can accept within the DSL."""
+
+    audio = "audio"
+    boolean = "boolean"
+    bytes = "bytes"
+    citation_document = "citation_document"
+    citation_url = "citation_url"
+    date = "date"
+    datetime = "datetime"
+    int = "int"
+    file = "file"
+    float = "float"
+    image = "image"
+    text = "text"
+    time = "time"
+    video = "video"
+    thinking = "thinking"
+
+
+class StepCardinality(str, Enum):
+    """Does this step emit 1 (one) or 0...N (many) items?"""
+
+    one = "one"
+    many = "many"
+
+
+ReferenceT = TypeVar("ReferenceT")
+
+
+class Reference(BaseModel, Generic[ReferenceT]):
+    """Represents a reference to another component by its ID."""
+
+    # model_config = PydanticConfigDict(extra="forbid")
+
+    ref: str = Field(..., alias="$ref")
+
+
+def _contains_reference_and_str(type_hint: Any) -> bool:
+    """Check if type contains both Reference and str in a union."""
+    # Get union args (handles Union, | syntax, and Optional)
+    origin = get_origin(type_hint)
+    if origin not in (Union, None) and not isinstance(
+        type_hint, types.UnionType
+    ):
+        return False
+
+    args = get_args(type_hint)
+    if not args:
+        return False
+
+    has_str = str in args
+    has_ref = any(
+        get_origin(arg) is Reference
+        or (hasattr(arg, "__mro__") and Reference in arg.__mro__)
+        for arg in args
+    )
+    return has_str and has_ref
+
+
+def _should_transform_field(type_hint: Any) -> tuple[bool, bool]:
+    """
+    Check if field should be transformed.
+    Returns: (should_transform, is_list)
+    """
+    # Check direct union: Reference[T] | str
+    if _contains_reference_and_str(type_hint):
+        return True, False
+
+    # Check list of union: list[Reference[T] | str]
+    origin = get_origin(type_hint)
+    if origin is list:
+        args = get_args(type_hint)
+        if args and _contains_reference_and_str(args[0]):
+            return True, True
+
+    # Check optional list: list[Reference[T] | str] | None
+    if origin is Union or isinstance(type_hint, types.UnionType):
+        for arg in get_args(type_hint):
+            if get_origin(arg) is list:
+                inner_args = get_args(arg)
+                if inner_args and _contains_reference_and_str(inner_args[0]):
+                    return True, True
+
+    return False, False
+
+
+class StrictBaseModel(BaseModel):
+    """Base model with extra fields forbidden."""
+
+    model_config = PydanticConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_string_references(cls, data: Any) -> Any:
+        """
+        Normalize string references to Reference objects before validation.
+
+        Transforms:
+        - `field: "ref_id"` -> `field: {"$ref": "ref_id"}`
+        - `field: ["ref1", "ref2"]` -> `field: [{"$ref": "ref1"}, {"$ref": "ref2"}]`
+
+        Only applies to fields typed as `Reference[T] | str` or `list[Reference[T] | str]`.
+        """
+        if not isinstance(data, dict):
+            return data
+
+        # Get type hints (evaluates ForwardRefs)
+        hints = typing.get_type_hints(cls)
+
+        # Transform fields
+        for field_name, field_value in data.items():
+            if field_name == "type" or field_name not in hints:
+                continue
+
+            should_transform, is_list = _should_transform_field(
+                hints[field_name]
+            )
+            if not should_transform:
+                continue
+
+            if is_list and isinstance(field_value, list):
+                data[field_name] = [
+                    {"$ref": item} if isinstance(item, str) else item
+                    for item in field_value
+                ]
+            elif not is_list and isinstance(field_value, str):
+                data[field_name] = {"$ref": field_value}
+
+        return data
+
+
+class BatchConfig(BaseModel):
+    """Configuration for batch execution.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    batch_size: int = Field(
+        default=25,
+        description="Max number of rows to send to a step at a time",
+        gt=0,
+    )
+
+
+class ConcurrencyConfig(BaseModel):
+    """Configuration for concurrent processing.
+
+    Attributes:
+        num_workers: Number of async workers for batch operations.
+    """
+
+    num_workers: int = Field(
+        default=1,
+        description="Number of async workers for batch operations",
+        gt=0,
+    )
+
+
+class BatchableStepMixin(BaseModel):
+    """A mixin for steps that support concurrent batch processing."""
+
+    batch_config: BatchConfig = Field(
+        default_factory=BatchConfig,
+        description="Configuration for processing the input stream in batches. If omitted, the step processes items one by one.",
+    )
+
+
+class CacheConfig(BaseModel):
+    directory: PathLike = Field(
+        default=pathlib.Path("./.qtype-cache"),
+        description="Base cache directory.",
+    )
+    namespace: Optional[str] = Field(
+        default=None, description="Logical namespace for cache keys."
+    )
+    on_error: Literal["Cache", "Drop"] = "Drop"
+    version: str = Field(
+        default="1.0", description="Bump to invalidate old cache."
+    )
+    compress: bool = Field(default=False, description="Compress stored data.")
+    ttl: Optional[int] = Field(
+        default=None, description="Optional time-to-live in seconds."
+    )
+
+
+class CachedStepMixin(BaseModel):
+    """A mixin for steps that support caching."""
+
+    cache_config: CacheConfig | None = Field(
+        default=None,
+        description="Configuration for caching step outputs. If omitted, caching is disabled.",
+    )
+
+
+class ConcurrentStepMixin(BaseModel):
+    """A mixin for steps that support concurrent processing."""
+
+    concurrency_config: ConcurrencyConfig = Field(
+        default_factory=ConcurrencyConfig,
+        description="Configuration for processing the input stream concurrently. If omitted, the step processes items sequentially.",
+    )

{qtype-0.0.16 → qtype-0.1.1}/qtype/cli.py

@@ -7,6 +7,10 @@ import importlib
 import logging
 from pathlib import Path
 
+from qtype.base.logging import get_logger
+
+logger = get_logger("application.facade")
+
 try:
     from importlib.metadata import entry_points
 except ImportError: