qtype 0.0.12__py3-none-any.whl → 0.1.7__py3-none-any.whl

qtype/commands/run.py

@@ -7,11 +7,12 @@ from __future__ import annotations
 import argparse
 import json
 import logging
+import warnings
 from pathlib import Path
 from typing import Any
 
-import magic
 import pandas as pd
+from pydantic.warnings import UnsupportedFieldAttributeWarning
 
 from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
@@ -19,14 +20,45 @@ from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
 logger = logging.getLogger(__name__)
 
 
+# Supress specific pydantic warnings that llamaindex needs to fix
+warnings.filterwarnings("ignore", category=UnsupportedFieldAttributeWarning)
+
+
+# supress qdrant logging
+for name in ["httpx", "urllib3", "qdrant_client", "opensearch"]:
+    logging.getLogger(name).setLevel(logging.WARNING)
+
+
 def read_data_from_file(file_path: str) -> pd.DataFrame:
     """
     Reads a file into a pandas DataFrame based on its MIME type.
     """
+    from pathlib import Path
+
+    import magic
+
     mime_type = magic.Magic(mime=True).from_file(file_path)
 
     if mime_type == "text/csv":
-        return pd.read_csv(file_path)
+        # TODO: Restore na values and convert to optional once we support them https://github.com/bazaarvoice/qtype/issues/101
+        df = pd.read_csv(file_path)
+        return df.fillna("")
+    elif mime_type == "text/plain":
+        # For text/plain, use file extension to determine format
+        file_ext = Path(file_path).suffix.lower()
+        if file_ext == ".csv":
+            # TODO: Restore na values and convert to optional once we support them https://github.com/bazaarvoice/qtype/issues/101
+            df = pd.read_csv(file_path)
+            return df.fillna("")
+        elif file_ext == ".json":
+            return pd.read_json(file_path)
+        else:
+            raise ValueError(
+                (
+                    f"Unsupported text/plain file extension: {file_ext}. "
+                    "Supported extensions: .csv, .json"
+                )
+            )
     elif mime_type == "application/json":
         return pd.read_json(file_path)
     elif mime_type in [
@@ -48,6 +80,8 @@ def run_flow(args: Any) -> None:
     Args:
         args: Arguments passed from the command line or calling context.
     """
+    import asyncio
+
     facade = QTypeFacade()
     spec_path = Path(args.spec)
 
@@ -65,53 +99,56 @@ def run_flow(args: Any) -> None:
                 logger.error(f"❌ Invalid JSON input: {e}")
                 return
 
-        # Execute the workflow using the facade
-        result = facade.execute_workflow(
-            spec_path, flow_name=args.flow, inputs=input, batch_config=None
+        # Execute the workflow using the facade (now async, returns DataFrame)
+        result_df = asyncio.run(
+            facade.execute_workflow(
+                spec_path,
+                flow_name=args.flow,
+                inputs=input,
+                show_progress=args.progress,
+            )
         )
 
         logger.info("✅ Flow execution completed successfully")
 
-        # Print results
-        if isinstance(result, pd.DataFrame):
-            logging.info("Output DataFrame:")
-            logging.info(result)
-        elif (
-            result
-            and hasattr(result, "__iter__")
-            and not isinstance(result, str)
-        ):
-            # If result is a list of variables or similar
-            try:
-                for item in result:
-                    if hasattr(item, "id") and hasattr(item, "value"):
-                        logger.info(f"Output {item.id}: {item.value}")
-                    else:
-                        logger.info(f"Result: {item}")
-            except TypeError:
-                logger.info(f"Result: {result}")
-        elif isinstance(result, str):
-            logger.info(f"Result: {result}")
+        # Display results
+        if len(result_df) > 0:
+            logger.info(f"Processed {len(result_df)} em")
+
+            # Remove 'row' and 'error' columns for display if all errors are None
+            display_df = result_df.copy()
+            if (
+                "error" in display_df.columns
+                and display_df["error"].isna().all()
+            ):
+                display_df = display_df.drop(columns=["error"])
+            if "row" in display_df.columns:
+                display_df = display_df.drop(columns=["row"])
+
+            if len(display_df) > 1:
+                logger.info(f"\nResults:\n{display_df[0:10].to_string()}\n...")
+            else:
+                # Print the first row with column_name: value one per line
+                fmt_str = []
+                for col, val in display_df.iloc[0].items():
+                    fmt_str.append(f"{col}: {val}")
+                fmt_str = "\n".join(fmt_str)
+                logger.info(f"\nResults:\n{fmt_str}")
+
+            # Save the output
+            if args.output:
+                # Save full DataFrame with row and error columns
+                result_df.to_parquet(args.output)
+                logger.info(f"Output saved to {args.output}")
         else:
             logger.info("Flow completed with no output")
 
-        # save the output
-        if isinstance(result, pd.DataFrame) and args.output:
-            result.to_parquet(args.output)
-            logger.info(f"Output DataFrame saved to {args.output}")
-        elif args.output:
-            with open(args.output, "w") as f:
-                json.dump(result, f, indent=2)
-            logger.info(f"Output saved to {args.output}")
-
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")
     except ValidationError as e:
         logger.error(f"❌ Validation failed: {e}")
     except InterpreterError as e:
         logger.error(f"❌ Execution failed: {e}")
-    except Exception as e:
-        logger.error(f"❌ Unexpected error: {e}", exc_info=True)
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:
@@ -153,6 +190,11 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
         default=None,
         help="Path to save output data. If input is a DataFrame, output will be saved as parquet. If single result, saved as JSON.",
     )
+    cmd_parser.add_argument(
+        "--progress",
+        action="store_true",
+        help="Show progress bars during flow execution.",
+    )
 
     cmd_parser.add_argument(
         "spec", type=str, help="Path to the QType YAML spec file."

qtype/commands/serve.py

@@ -6,61 +6,91 @@ from __future__ import annotations
 
 import argparse
 import logging
+import os
 from pathlib import Path
 from typing import Any
 
 import uvicorn
 
-from qtype.application.facade import QTypeFacade
-from qtype.base.exceptions import LoadError, ValidationError
+from qtype.base.exceptions import ValidationError
+from qtype.semantic.loader import load
+from qtype.semantic.model import Application
 
 logger = logging.getLogger(__name__)
 
 
-def serve(args: Any) -> None:
+def create_api_app() -> Any:
+    """Factory function to create FastAPI app.
+
+    Returns:
+        FastAPI application instance.
+
+    Raises:
+        RuntimeError: If QTYPE_SPEC_PATH not set in environment.
+        ValidationError: If spec is not an Application document.
+    """
+    from qtype.interpreter.api import APIExecutor
+
+    spec_path_str = os.environ["_QTYPE_SPEC_PATH"]
+
+    spec_path = Path(spec_path_str)
+    logger.info(f"Loading spec: {spec_path}")
+
+    semantic_model, _ = load(spec_path)
+    if not isinstance(semantic_model, Application):
+        raise ValidationError("Can only serve Application documents")
+
+    logger.info(f"✅ Successfully loaded spec: {spec_path}")
+
+    # Derive name from spec filename
+    name = spec_path.name.replace(".qtype.yaml", "").replace("_", " ").title()
+
+    # Get host/port from environment (set by uvicorn)
+    host = os.environ["_QTYPE_HOST"]
+    port = int(os.environ["_QTYPE_PORT"])
+
+    # Create server info for OpenAPI spec
+    servers = [
+        {
+            "url": f"http://{host}:{port}",
+            "description": "Development server",
+        }
+    ]
+
+    api_executor = APIExecutor(semantic_model, host, port)
+    return api_executor.create_app(
+        name=name,
+        ui_enabled=True,
+        servers=servers,
+    )
+
+
+def serve(args: argparse.Namespace) -> None:
     """Run a QType YAML spec file as an API.
 
     Args:
-        args: Arguments passed from the command line or calling context.
+        args: Arguments passed from the command line.
     """
-    facade = QTypeFacade()
-
-    try:
-        # Use facade to load and validate the document
-        spec_path = Path(args.spec)
-        logger.info(f"Loading and validating spec: {spec_path}")
-
-        semantic_model, type_registry = facade.load_semantic_model(spec_path)
-        logger.info(f"✅ Successfully loaded spec: {spec_path}")
-
-        # Import APIExecutor and create the FastAPI app
-        from qtype.interpreter.api import APIExecutor
-
-        # Get the name from the spec filename
-        name = (
-            spec_path.name.replace(".qtype.yaml", "").replace("_", " ").title()
-        )
-
-        logger.info(f"Starting server for: {name}")
-        api_executor = APIExecutor(semantic_model)
-        fastapi_app = api_executor.create_app(
-            name=name, ui_enabled=not args.disable_ui
-        )
-
-        # Start the server
-        uvicorn.run(
-            fastapi_app, host=args.host, port=args.port, log_level="info"
-        )
-
-    except LoadError as e:
-        logger.error(f"❌ Failed to load document: {e}")
-        exit(1)
-    except ValidationError as e:
-        logger.error(f"❌ Validation failed: {e}")
-        exit(1)
-    except Exception as e:
-        logger.error(f"❌ Unexpected error starting server: {e}")
-        exit(1)
+    # Set environment variables for factory function
+    os.environ["_QTYPE_SPEC_PATH"] = args.spec
+    os.environ["_QTYPE_HOST"] = args.host
+    os.environ["_QTYPE_PORT"] = str(args.port)
+
+    logger.info(
+        f"Starting server on {args.host}:{args.port}"
+        f"{' (reload enabled)' if args.reload else ''}"
+    )
+
+    # Use factory mode with import string
+    uvicorn.run(
+        "qtype.commands.serve:create_api_app",
+        factory=True,
+        host=args.host,
+        port=args.port,
+        log_level="info",
+        reload=args.reload,
+        reload_includes=[args.spec] if args.reload else None,
+    )
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:
@@ -76,9 +106,9 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
     cmd_parser.add_argument("-p", "--port", type=int, default=8000)
     cmd_parser.add_argument("-H", "--host", type=str, default="localhost")
     cmd_parser.add_argument(
-        "--disable-ui",
+        "--reload",
         action="store_true",
-        help="Disable the UI for the QType application.",
+        help="Enable auto-reload on code changes (default: False).",
     )
     cmd_parser.set_defaults(func=serve)
 

qtype/commands/validate.py

@@ -10,8 +10,12 @@ import sys
 from pathlib import Path
 from typing import Any
 
-from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import LoadError, SemanticError, ValidationError
+from qtype.dsl.linker import DuplicateComponentError, ReferenceNotFoundError
+from qtype.dsl.loader import YAMLLoadError
+from qtype.dsl.model import Application as DSLApplication
+from qtype.dsl.model import Document
+from qtype.semantic.loader import load
 
 logger = logging.getLogger(__name__)
 
@@ -26,33 +30,52 @@ def main(args: Any) -> None:
     Exits:
         Exits with code 1 if validation fails.
     """
-    facade = QTypeFacade()
     spec_path = Path(args.spec)
 
     try:
-        # Use the facade for validation - it will raise exceptions on errors
-        loaded_data = facade.load_and_validate(spec_path)
+        # Load and validate the document (works for all document types)
+        # This includes: YAML parsing, Pydantic validation, linking, and semantic checks
+        loaded_data, custom_types = load(spec_path)
         logger.info("✅ Validation successful - document is valid.")
 
-        # If printing is requested, load and print the document
-        if args.print:
-            try:
-                print(loaded_data.model_dump_json(indent=2, exclude_none=True))  # type: ignore
-            except Exception as e:
-                logger.warning(f"Could not print document: {e}")
-
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")
         sys.exit(1)
+    except YAMLLoadError as e:
+        # YAML syntax errors
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except DuplicateComponentError as e:
+        # Duplicate ID errors during linking
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except ReferenceNotFoundError as e:
+        # Reference resolution errors during linking
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except ValueError as e:
+        # Pydantic validation errors from parse_document
+        logger.error(f"❌ {e}")
+        sys.exit(1)
     except ValidationError as e:
         logger.error(f"❌ Validation failed: {e}")
         sys.exit(1)
     except SemanticError as e:
         logger.error(f"❌ Semantic validation failed: {e}")
         sys.exit(1)
-    except Exception as e:
-        logger.error(f"❌ Unexpected error during validation: {e}")
-        sys.exit(1)
+
+    # If printing is requested, load and print the document
+    if args.print:
+        from pydantic_yaml import to_yaml_str
+
+        # Wrap in Document if it's a DSL Application
+        if isinstance(loaded_data, DSLApplication):
+            wrapped = Document(root=loaded_data)
+        else:
+            wrapped = loaded_data
+        logging.info(
+            to_yaml_str(wrapped, exclude_unset=True, exclude_none=True)
+        )
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:

qtype/commands/visualize.py

@@ -6,13 +6,17 @@ from __future__ import annotations
 
 import argparse
 import logging
+import shutil
+import subprocess
 import tempfile
 import webbrowser
 from pathlib import Path
 from typing import Any
 
-from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import LoadError, ValidationError
+from qtype.semantic.loader import load
+from qtype.semantic.model import Application
+from qtype.semantic.visualize import visualize_application
 
 logger = logging.getLogger(__name__)
 
@@ -27,12 +31,13 @@ def main(args: Any) -> None:
     Exits:
         Exits with code 1 if visualization fails.
     """
-    facade = QTypeFacade()
     spec_path = Path(args.spec)
 
     try:
-        # Generate visualization using the facade
-        mermaid_content = facade.visualize_application(spec_path)
+        # Load and generate visualization
+        semantic_model, _ = load(spec_path)
+        assert isinstance(semantic_model, Application)
+        mermaid_content = visualize_application(semantic_model)
 
         if args.output:
             # Write to file
@@ -41,26 +46,45 @@ def main(args: Any) -> None:
             logger.info(f"✅ Visualization saved to {output_path}")
 
         if not args.no_display:
-            # Create temporary HTML file and open in browser
+            # Check if mmdc is available
+            if shutil.which("mmdc") is None:
+                logger.error(
+                    "❌ mmdc command not found. Please install mermaid-cli."
+                )
+                logger.info(
+                    "Install with: npm install -g @mermaid-js/mermaid-cli"
+                )
+                exit(1)
+
+            # Create temporary directory and run mmdc from within it
             try:
-                import mermaid as md  # type: ignore[import-untyped]
-
-                mm = md.Mermaid(mermaid_content)
-                html_content = mm._repr_html_()
-
-                with tempfile.NamedTemporaryFile(
-                    mode="w", suffix=".html", delete=False, encoding="utf-8"
-                ) as f:
-                    f.write(html_content)
-                    temp_file = f.name
-
-                logger.info(f"Opening visualization in browser: {temp_file}")
-                webbrowser.open(f"file://{temp_file}")
-            except ImportError:
-                logger.warning(
-                    "❌ Mermaid library not installed. Cannot display in browser."
+                # Create a temporary directory
+                temp_dir = tempfile.mkdtemp()
+                temp_dir_path = Path(temp_dir)
+
+                # Write mermaid file with simple names in the temp directory
+                mmd_file_path = temp_dir_path / "diagram.mmd"
+                svg_file_path = temp_dir_path / "diagram.svg"
+
+                mmd_file_path.write_text(mermaid_content, encoding="utf-8")
+
+                # Run mmdc from within the temporary directory
+                subprocess.run(
+                    ["mmdc", "-i", "diagram.mmd", "-o", "diagram.svg"],
+                    cwd=temp_dir,
+                    capture_output=True,
+                    text=True,
+                    check=True,
                 )
-                logger.info("Install with: pip install mermaid")
+
+                logger.info(
+                    f"Opening visualization in browser: {svg_file_path}"
+                )
+                webbrowser.open(f"file://{svg_file_path}")
+
+            except subprocess.CalledProcessError as e:
+                logger.error(f"❌ Failed to generate SVG: {e.stderr}")
+                exit(1)
 
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")
@@ -68,9 +92,6 @@ def main(args: Any) -> None:
     except ValidationError as e:
         logger.error(f"❌ Visualization failed: {e}")
         exit(1)
-    except Exception as e:
-        logger.error(f"❌ Unexpected error: {e}")
-        exit(1)
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:

qtype/dsl/__init__.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from .base_types import *  # noqa: F403
-from .domain_types import *  # noqa: F403
-from .model import *  # noqa: F403
+# Import all public symbols from submodules
+# The star import is intentional here - it's controlled by __all__ in each module
+from qtype.base.types import Reference  # noqa: F401
 
-# Note: Validation logic has been moved to qtype.semantic package
-# to avoid circular dependencies
+from .domain_types import *  # noqa: F403, F401
+from .loader import YAMLLoadError  # noqa: F401
+from .model import *  # noqa: F403, F401

qtype/dsl/custom_types.py

@@ -2,7 +2,7 @@ from typing import Any, ForwardRef, Type, Union
 
 from pydantic import BaseModel, create_model
 
-from qtype.application.converters.types import PRIMITIVE_TO_PYTHON_TYPE
+from qtype.dsl.types import PRIMITIVE_TO_PYTHON_TYPE
 
 # --- This would be in your interpreter's logic ---
 

qtype/dsl/domain_types.py

@@ -5,7 +5,7 @@ from typing import Any
 
 from pydantic import Field
 
-from qtype.dsl.base_types import PrimitiveTypeEnum, StrictBaseModel
+from qtype.base.types import PrimitiveTypeEnum, StrictBaseModel
 
 
 class Embedding(StrictBaseModel):
@@ -14,11 +14,12 @@ class Embedding(StrictBaseModel):
     vector: list[float] = Field(
         ..., description="The vector representation of the embedding."
     )
-    source_text: str | None = Field(
-        None, description="The original text that was embedded."
+    content: Any | None = Field(
+        None, description="The original content that was embedded."
     )
-    metadata: dict[str, str] | None = Field(
-        None, description="Optional metadata associated with the embedding."
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Metadata associated with the embedding.",
     )
 
 
@@ -57,3 +58,83 @@ class ChatMessage(StrictBaseModel):
         ...,
         description="The content blocks of the chat message, which can include text, images, or other media.",
     )
+
+
+class RAGDocument(StrictBaseModel):
+    """A standard, built-in representation of a document used in Retrieval-Augmented Generation (RAG)."""
+
+    content: Any = Field(..., description="The main content of the document.")
+    file_id: str = Field(..., description="An unique identifier for the file.")
+    file_name: str = Field(..., description="The name of the file.")
+    uri: str | None = Field(
+        None, description="The URI where the document can be found."
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Metadata associated with the document.",
+    )
+    type: PrimitiveTypeEnum = Field(
+        ...,
+        description="The type of the document content (e.g., 'text', 'image').",
+    )
+
+
+class RAGChunk(Embedding):
+    """A standard, built-in representation of a chunk of a document used in Retrieval-Augmented Generation (RAG)."""
+
+    chunk_id: str = Field(
+        ..., description="An unique identifier for the chunk."
+    )
+    document_id: str = Field(
+        ..., description="The identifier of the parent document."
+    )
+    vector: list[float] | None = Field(
+        None, description="Optional vector embedding for the chunk."
+    )
+
+
+class SearchResult(StrictBaseModel):
+    """A standard, built-in representation of a search result."""
+
+    content: Any = Field(..., description="The content of the search result.")
+    doc_id: str = Field(
+        ...,
+        description="The identifier of the document from which the result was retrieved.",
+    )
+    score: float = Field(
+        ...,
+        description="The relevance score of the search result with respect to the query.",
+    )
+
+
+class RAGSearchResult(SearchResult):
+    """A standard, built-in representation of a search result from a RAG vector search.
+
+    Note: doc_id is duplicated from content.document_id for convenience.
+    """
+
+    content: RAGChunk = Field(
+        ..., description="The RAG chunk returned as a search result."
+    )
+    doc_id: str = Field(
+        ...,
+        description="The document ID (duplicated from content.document_id).",
+    )
+    score: float = Field(
+        ...,
+        description="The similarity score of the chunk with respect to the query.",
+    )
+
+
+class AggregateStats(StrictBaseModel):
+    """A standard, built-in representation of aggregate statistics."""
+
+    num_successful: int = Field(
+        ..., description="The count of successful messages processed."
+    )
+    num_failed: int = Field(
+        ..., description="The count of failed messages processed."
+    )
+    num_total: int = Field(
+        ..., description="The total count of messages processed."
+    )