qtype 0.0.16__py3-none-any.whl → 0.1.1__py3-none-any.whl

qtype/commands/generate.py

@@ -92,6 +92,17 @@ def generate_schema(args: argparse.Namespace) -> None:
     if "$defs" not in schema:
         schema["$defs"] = {}
 
+    # Note: Custom YAML tags (!include, !include_raw) and environment variable
+    # substitution (${VAR}) are handled by the QType YAML loader at parse time,
+    # not by JSON Schema validation. We define them in $defs for documentation
+    # purposes, but we don't apply them to string fields since:
+    # 1. They would cause false positives (e.g., "localhost" matching as valid)
+    # 2. The YAML loader processes these before schema validation occurs
+    # 3. After YAML loading, the schema sees the resolved/substituted values
+    #
+    # Schema validation happens on the post-processed document structure,
+    # so we don't need to (and shouldn't) validate the raw YAML tag syntax.
+
     # Define custom YAML tags used by QType loader
     schema["$defs"]["qtype_include_tag"] = {
         "type": "string",
@@ -111,29 +122,6 @@ def generate_schema(args: argparse.Namespace) -> None:
         "description": "String with environment variable substitution using ${VAR_NAME} or ${VAR_NAME:default} syntax",
     }
 
-    # Add these custom patterns to string types throughout the schema
-    def add_custom_patterns(obj):
-        if isinstance(obj, dict):
-            if obj.get("type") == "string" and "anyOf" not in obj:
-                # Add anyOf to allow either regular strings or custom tag patterns
-                original_obj = obj.copy()
-                obj.clear()
-                obj["anyOf"] = [
-                    original_obj,
-                    {"$ref": "#/$defs/qtype_include_tag"},
-                    {"$ref": "#/$defs/qtype_include_raw_tag"},
-                    {"$ref": "#/$defs/qtype_env_var"},
-                ]
-            else:
-                for value in obj.values():
-                    add_custom_patterns(value)
-        elif isinstance(obj, list):
-            for item in obj:
-                add_custom_patterns(item)
-
-    # Apply custom patterns to the schema
-    add_custom_patterns(schema)
-
     output = json.dumps(schema, indent=2)
     output_path: Optional[str] = getattr(args, "output", None)
     if output_path:
@@ -198,6 +186,14 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
         import networkx  # noqa: F401
         import ruff  # type: ignore[import-untyped]  # noqa: F401
 
+        has_semantic_deps = True
+    except ImportError:
+        logger.warning(
+            "NetworkX or Ruff is not installed. Skipping semantic model generation."
+        )
+        has_semantic_deps = False
+
+    if has_semantic_deps:
         from qtype.semantic.generate import generate_semantic_model
 
         semantic_parser = generate_subparsers.add_parser(
@@ -212,7 +208,3 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
             help="Output file for the semantic model (default: stdout)",
         )
         semantic_parser.set_defaults(func=generate_semantic_model)
-    except ImportError:
-        logger.debug(
-            "NetworkX or Ruff is not installed. Skipping semantic model generation."
-        )

qtype/commands/run.py

@@ -7,11 +7,12 @@ from __future__ import annotations
 import argparse
 import json
 import logging
+import warnings
 from pathlib import Path
 from typing import Any
 
-import magic
 import pandas as pd
+from pydantic.warnings import UnsupportedFieldAttributeWarning
 
 from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
@@ -19,14 +20,41 @@ from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
 logger = logging.getLogger(__name__)
 
 
+# Supress specific pydantic warnings that llamaindex needs to fix
+warnings.filterwarnings("ignore", category=UnsupportedFieldAttributeWarning)
+
+
+# supress qdrant logging
+for name in ["httpx", "urllib3", "qdrant_client"]:
+    logging.getLogger(name).setLevel(logging.WARNING)
+
+
 def read_data_from_file(file_path: str) -> pd.DataFrame:
     """
     Reads a file into a pandas DataFrame based on its MIME type.
     """
+    from pathlib import Path
+
+    import magic
+
     mime_type = magic.Magic(mime=True).from_file(file_path)
 
     if mime_type == "text/csv":
         return pd.read_csv(file_path)
+    elif mime_type == "text/plain":
+        # For text/plain, use file extension to determine format
+        file_ext = Path(file_path).suffix.lower()
+        if file_ext == ".csv":
+            return pd.read_csv(file_path)
+        elif file_ext == ".json":
+            return pd.read_json(file_path)
+        else:
+            raise ValueError(
+                (
+                    f"Unsupported text/plain file extension: {file_ext}. "
+                    "Supported extensions: .csv, .json"
+                )
+            )
     elif mime_type == "application/json":
         return pd.read_json(file_path)
     elif mime_type in [
@@ -48,6 +76,8 @@ def run_flow(args: Any) -> None:
     Args:
         args: Arguments passed from the command line or calling context.
     """
+    import asyncio
+
     facade = QTypeFacade()
     spec_path = Path(args.spec)
 
@@ -65,54 +95,56 @@ def run_flow(args: Any) -> None:
                 logger.error(f"❌ Invalid JSON input: {e}")
                 return
 
-        # Execute the workflow using the facade
-        result = facade.execute_workflow(
-            spec_path, flow_name=args.flow, inputs=input, batch_config=None
+        # Execute the workflow using the facade (now async, returns DataFrame)
+        result_df = asyncio.run(
+            facade.execute_workflow(
+                spec_path,
+                flow_name=args.flow,
+                inputs=input,
+                show_progress=args.progress,
+            )
         )
 
         logger.info("✅ Flow execution completed successfully")
 
-        # Print results
-        if isinstance(result, pd.DataFrame):
-            logging.info("Output DataFrame:")
-            logging.info(result)
-        elif (
-            result
-            and hasattr(result, "__iter__")
-            and not isinstance(result, str)
-        ):
-            # If result is a list of variables or similar
-            try:
-                for item in result:
-                    if hasattr(item, "id") and hasattr(item, "value"):
-                        logger.info(f"Output {item.id}: {item.value}")
-                    else:
-                        logger.info(f"Result: {item}")
-            except TypeError:
-                logger.info(f"Result: {result}")
-        elif isinstance(result, str):
-            logger.info(f"Result: {result}")
+        # Display results
+        if len(result_df) > 0:
+            logger.info(f"Processed {len(result_df)} em")
+
+            # Remove 'row' and 'error' columns for display if all errors are None
+            display_df = result_df.copy()
+            if (
+                "error" in display_df.columns
+                and display_df["error"].isna().all()
+            ):
+                display_df = display_df.drop(columns=["error"])
+            if "row" in display_df.columns:
+                display_df = display_df.drop(columns=["row"])
+
+            if len(display_df) > 1:
+                logger.info(f"\nResults:\n{display_df[0:10].to_string()}\n...")
+            else:
+                # Print the first row with column_name: value one per line
+                fmt_str = []
+                for col, val in display_df.iloc[0].items():
+                    fmt_str.append(f"{col}: {val}")
+                fmt_str = "\n".join(fmt_str)
+                logger.info(f"\nResults:\n{fmt_str}")
+
+            # Save the output
+            if args.output:
+                # Save full DataFrame with row and error columns
+                result_df.to_parquet(args.output)
+                logger.info(f"Output saved to {args.output}")
         else:
             logger.info("Flow completed with no output")
 
-        # save the output
-        if isinstance(result, pd.DataFrame) and args.output:
-            result.to_parquet(args.output)
-            logger.info(f"Output DataFrame saved to {args.output}")
-        elif args.output:
-            with open(args.output, "w") as f:
-                json.dump(result, f, indent=2)
-            logger.info(f"Output saved to {args.output}")
-
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")
     except ValidationError as e:
         logger.error(f"❌ Validation failed: {e}")
     except InterpreterError as e:
         logger.error(f"❌ Execution failed: {e}")
-    except Exception as e:
-        logger.error(f"❌ Unexpected error: {e}", exc_info=True)
-        pass
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:
@@ -154,6 +186,11 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
         default=None,
         help="Path to save output data. If input is a DataFrame, output will be saved as parquet. If single result, saved as JSON.",
     )
+    cmd_parser.add_argument(
+        "--progress",
+        action="store_true",
+        help="Show progress bars during flow execution.",
+    )
 
     cmd_parser.add_argument(
         "spec", type=str, help="Path to the QType YAML spec file."

qtype/commands/serve.py

@@ -6,71 +6,91 @@ from __future__ import annotations
 
 import argparse
 import logging
+import os
 from pathlib import Path
 from typing import Any
 
 import uvicorn
 
-from qtype.application.facade import QTypeFacade
-from qtype.base.exceptions import LoadError, ValidationError
+from qtype.base.exceptions import ValidationError
+from qtype.semantic.loader import load
+from qtype.semantic.model import Application
 
 logger = logging.getLogger(__name__)
 
 
-def serve(args: Any) -> None:
+def create_api_app() -> Any:
+    """Factory function to create FastAPI app.
+
+    Returns:
+        FastAPI application instance.
+
+    Raises:
+        RuntimeError: If QTYPE_SPEC_PATH not set in environment.
+        ValidationError: If spec is not an Application document.
+    """
+    from qtype.interpreter.api import APIExecutor
+
+    spec_path_str = os.environ["_QTYPE_SPEC_PATH"]
+
+    spec_path = Path(spec_path_str)
+    logger.info(f"Loading spec: {spec_path}")
+
+    semantic_model, _ = load(spec_path)
+    if not isinstance(semantic_model, Application):
+        raise ValidationError("Can only serve Application documents")
+
+    logger.info(f"✅ Successfully loaded spec: {spec_path}")
+
+    # Derive name from spec filename
+    name = spec_path.name.replace(".qtype.yaml", "").replace("_", " ").title()
+
+    # Get host/port from environment (set by uvicorn)
+    host = os.environ["_QTYPE_HOST"]
+    port = int(os.environ["_QTYPE_PORT"])
+
+    # Create server info for OpenAPI spec
+    servers = [
+        {
+            "url": f"http://{host}:{port}",
+            "description": "Development server",
+        }
+    ]
+
+    api_executor = APIExecutor(semantic_model, host, port)
+    return api_executor.create_app(
+        name=name,
+        ui_enabled=True,
+        servers=servers,
+    )
+
+
+def serve(args: argparse.Namespace) -> None:
     """Run a QType YAML spec file as an API.
 
     Args:
-        args: Arguments passed from the command line or calling context.
+        args: Arguments passed from the command line.
     """
-    facade = QTypeFacade()
-
-    try:
-        # Use facade to load and validate the document
-        spec_path = Path(args.spec)
-        logger.info(f"Loading and validating spec: {spec_path}")
-
-        semantic_model, type_registry = facade.load_semantic_model(spec_path)
-        facade.telemetry(semantic_model)
-        logger.info(f"✅ Successfully loaded spec: {spec_path}")
-
-        # Import APIExecutor and create the FastAPI app
-        from qtype.interpreter.api import APIExecutor
-
-        # Get the name from the spec filename
-        name = (
-            spec_path.name.replace(".qtype.yaml", "").replace("_", " ").title()
-        )
-
-        logger.info(f"Starting server for: {name}")
-        api_executor = APIExecutor(semantic_model)
-
-        # Create server info for OpenAPI spec
-        servers = [
-            {
-                "url": f"http://{args.host}:{args.port}",
-                "description": "Development server",
-            }
-        ]
-
-        fastapi_app = api_executor.create_app(
-            name=name, ui_enabled=not args.disable_ui, servers=servers
-        )
-
-        # Start the server
-        uvicorn.run(
-            fastapi_app, host=args.host, port=args.port, log_level="info"
-        )
-
-    except LoadError as e:
-        logger.error(f"❌ Failed to load document: {e}")
-        exit(1)
-    except ValidationError as e:
-        logger.error(f"❌ Validation failed: {e}")
-        exit(1)
-    except Exception as e:
-        logger.error(f"❌ Unexpected error starting server: {e}")
-        exit(1)
+    # Set environment variables for factory function
+    os.environ["_QTYPE_SPEC_PATH"] = args.spec
+    os.environ["_QTYPE_HOST"] = args.host
+    os.environ["_QTYPE_PORT"] = str(args.port)
+
+    logger.info(
+        f"Starting server on {args.host}:{args.port}"
+        f"{' (reload enabled)' if args.reload else ''}"
+    )
+
+    # Use factory mode with import string
+    uvicorn.run(
+        "qtype.commands.serve:create_api_app",
+        factory=True,
+        host=args.host,
+        port=args.port,
+        log_level="info",
+        reload=args.reload,
+        reload_includes=[args.spec] if args.reload else None,
+    )
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:
@@ -86,9 +106,9 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
     cmd_parser.add_argument("-p", "--port", type=int, default=8000)
     cmd_parser.add_argument("-H", "--host", type=str, default="localhost")
     cmd_parser.add_argument(
-        "--disable-ui",
+        "--reload",
         action="store_true",
-        help="Disable the UI for the QType application.",
+        help="Enable auto-reload on code changes (default: False).",
     )
     cmd_parser.set_defaults(func=serve)
 

qtype/commands/validate.py

@@ -10,9 +10,12 @@ import sys
 from pathlib import Path
 from typing import Any
 
-from qtype import dsl
-from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import LoadError, SemanticError, ValidationError
+from qtype.dsl.linker import DuplicateComponentError, ReferenceNotFoundError
+from qtype.dsl.loader import YAMLLoadError
+from qtype.dsl.model import Application as DSLApplication
+from qtype.dsl.model import Document
+from qtype.semantic.loader import load
 
 logger = logging.getLogger(__name__)
 
@@ -27,19 +30,33 @@ def main(args: Any) -> None:
     Exits:
         Exits with code 1 if validation fails.
     """
-    facade = QTypeFacade()
     spec_path = Path(args.spec)
 
     try:
-        # Use the facade for validation - it will raise exceptions on errors
-        loaded_data, custom_types = facade.load_dsl_document(spec_path)
-        if isinstance(loaded_data, dsl.Application):
-            loaded_data, custom_types = facade.load_semantic_model(spec_path)
+        # Load and validate the document (works for all document types)
+        # This includes: YAML parsing, Pydantic validation, linking, and semantic checks
+        loaded_data, custom_types = load(spec_path)
         logger.info("✅ Validation successful - document is valid.")
 
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")
         sys.exit(1)
+    except YAMLLoadError as e:
+        # YAML syntax errors
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except DuplicateComponentError as e:
+        # Duplicate ID errors during linking
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except ReferenceNotFoundError as e:
+        # Reference resolution errors during linking
+        logger.error(f"❌ {e}")
+        sys.exit(1)
+    except ValueError as e:
+        # Pydantic validation errors from parse_document
+        logger.error(f"❌ {e}")
+        sys.exit(1)
     except ValidationError as e:
         logger.error(f"❌ Validation failed: {e}")
         sys.exit(1)
@@ -49,7 +66,16 @@ def main(args: Any) -> None:
 
     # If printing is requested, load and print the document
     if args.print:
-        logging.info(facade.convert_document(loaded_data))  # type: ignore
+        from pydantic_yaml import to_yaml_str
+
+        # Wrap in Document if it's a DSL Application
+        if isinstance(loaded_data, DSLApplication):
+            wrapped = Document(root=loaded_data)
+        else:
+            wrapped = loaded_data
+        logging.info(
+            to_yaml_str(wrapped, exclude_unset=True, exclude_none=True)
+        )
 
 
 def parser(subparsers: argparse._SubParsersAction) -> None:

qtype/commands/visualize.py

@@ -6,13 +6,17 @@ from __future__ import annotations
 
 import argparse
 import logging
+import shutil
+import subprocess
 import tempfile
 import webbrowser
 from pathlib import Path
 from typing import Any
 
-from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import LoadError, ValidationError
+from qtype.semantic.loader import load
+from qtype.semantic.model import Application
+from qtype.semantic.visualize import visualize_application
 
 logger = logging.getLogger(__name__)
 
@@ -27,12 +31,13 @@ def main(args: Any) -> None:
     Exits:
         Exits with code 1 if visualization fails.
     """
-    facade = QTypeFacade()
     spec_path = Path(args.spec)
 
     try:
-        # Generate visualization using the facade
-        mermaid_content = facade.visualize_application(spec_path)
+        # Load and generate visualization
+        semantic_model, _ = load(spec_path)
+        assert isinstance(semantic_model, Application)
+        mermaid_content = visualize_application(semantic_model)
 
         if args.output:
             # Write to file
@@ -41,26 +46,45 @@ def main(args: Any) -> None:
             logger.info(f"✅ Visualization saved to {output_path}")
 
         if not args.no_display:
-            # Create temporary HTML file and open in browser
+            # Check if mmdc is available
+            if shutil.which("mmdc") is None:
+                logger.error(
+                    "❌ mmdc command not found. Please install mermaid-cli."
+                )
+                logger.info(
+                    "Install with: npm install -g @mermaid-js/mermaid-cli"
+                )
+                exit(1)
+
+            # Create temporary directory and run mmdc from within it
             try:
-                import mermaid as md  # type: ignore[import-untyped]
-
-                mm = md.Mermaid(mermaid_content)
-                html_content = mm._repr_html_()
-
-                with tempfile.NamedTemporaryFile(
-                    mode="w", suffix=".html", delete=False, encoding="utf-8"
-                ) as f:
-                    f.write(html_content)
-                    temp_file = f.name
-
-                logger.info(f"Opening visualization in browser: {temp_file}")
-                webbrowser.open(f"file://{temp_file}")
-            except ImportError:
-                logger.warning(
-                    "❌ Mermaid library not installed. Cannot display in browser."
+                # Create a temporary directory
+                temp_dir = tempfile.mkdtemp()
+                temp_dir_path = Path(temp_dir)
+
+                # Write mermaid file with simple names in the temp directory
+                mmd_file_path = temp_dir_path / "diagram.mmd"
+                svg_file_path = temp_dir_path / "diagram.svg"
+
+                mmd_file_path.write_text(mermaid_content, encoding="utf-8")
+
+                # Run mmdc from within the temporary directory
+                subprocess.run(
+                    ["mmdc", "-i", "diagram.mmd", "-o", "diagram.svg"],
+                    cwd=temp_dir,
+                    capture_output=True,
+                    text=True,
+                    check=True,
                 )
-                logger.info("Install with: pip install mermaid")
+
+                logger.info(
+                    f"Opening visualization in browser: {svg_file_path}"
+                )
+                webbrowser.open(f"file://{svg_file_path}")
+
+            except subprocess.CalledProcessError as e:
+                logger.error(f"❌ Failed to generate SVG: {e.stderr}")
+                exit(1)
 
     except LoadError as e:
         logger.error(f"❌ Failed to load document: {e}")

qtype/dsl/__init__.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from .base_types import *  # noqa: F403
-from .domain_types import *  # noqa: F403
-from .model import *  # noqa: F403
+# Import all public symbols from submodules
+# The star import is intentional here - it's controlled by __all__ in each module
+from qtype.base.types import Reference  # noqa: F401
 
-# Note: Validation logic has been moved to qtype.semantic package
-# to avoid circular dependencies
+from .domain_types import *  # noqa: F403, F401
+from .loader import YAMLLoadError  # noqa: F401
+from .model import *  # noqa: F403, F401

qtype/dsl/custom_types.py

@@ -2,7 +2,7 @@ from typing import Any, ForwardRef, Type, Union
 
 from pydantic import BaseModel, create_model
 
-from qtype.application.converters.types import PRIMITIVE_TO_PYTHON_TYPE
+from qtype.dsl.types import PRIMITIVE_TO_PYTHON_TYPE
 
 # --- This would be in your interpreter's logic ---