qtype 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

qtype/mcp/server.py

@@ -1,22 +1,25 @@
 from __future__ import annotations
 
 import json
-import re
+import tempfile
 from functools import lru_cache
 from importlib.resources import files
 from pathlib import Path
 from typing import Any
 
+import tantivy
 from mcp.server.fastmcp import FastMCP
 from pydantic import BaseModel
 
+from qtype.base.resources import get_docs_resource, get_examples_resource
 from qtype.commands.convert import convert_to_yaml
 
 # Initialize FastMCP server
 mcp = FastMCP("qtype", host="0.0.0.0")
 
-# Regex for pymdownx snippets: --8<-- "path/to/file"
-SNIPPET_REGEX = re.compile(r'--8<--\s+"([^"]+)"')
+# Get resource directories from base layer
+_docs_resource = get_docs_resource()
+_examples_resource = get_examples_resource()
 
 
 # ============================================================================
@@ -24,24 +27,6 @@ SNIPPET_REGEX = re.compile(r'--8<--\s+"([^"]+)"')
 # ============================================================================
 
 
-def _get_docs_path() -> Path:
-    """Get the path to the documentation directory.
-
-    Returns:
-        Path to the docs directory, trying installed package first,
-        then falling back to development path.
-    """
-    try:
-        # Try to get from installed package
-        docs_root = files("qtype") / "docs"
-        # Check if it exists by trying to iterate
-        list(docs_root.iterdir())
-        return Path(str(docs_root))
-    except (FileNotFoundError, AttributeError, TypeError):
-        # Fall back to development path
-        return Path(__file__).parent.parent.parent / "docs"
-
-
 @lru_cache(maxsize=1)
 def _load_schema() -> dict[str, Any]:
     """Load the QType schema JSON file once and cache it.
@@ -55,7 +40,7 @@ def _load_schema() -> dict[str, Any]:
     """
     # Try to load from installed package data first
     try:
-        schema_file = files("qtype") / "qtype.schema.json"
+        schema_file = files("qtype") / "schema" / "qtype.schema.json"
         schema_text = schema_file.read_text(encoding="utf-8")
         return json.loads(schema_text)
     except (FileNotFoundError, AttributeError):
@@ -67,38 +52,88 @@ def _load_schema() -> dict[str, Any]:
             return json.load(f)
 
 
-def resolve_snippets(content: str, base_path: Path) -> str:
-    """
-    Recursively finds and replaces MkDocs snippets in markdown content.
-    Mimics the behavior of pymdownx.snippets.
+@lru_cache(maxsize=1)
+def _build_search_index() -> tantivy.Index:
+    """Build and cache a Tantivy search index for docs and examples.
 
-    Args:
-        content: The markdown content to process
-        base_path: Path to the file being processed (used to resolve relative paths)
+    Returns:
+        A Tantivy Index containing all documentation markdown files
+        and example YAML files.
+
+    Raises:
+        Exception: If index building fails.
     """
-    docs_root = _get_docs_path()
-    project_root = docs_root.parent
-
-    def replace_match(match):
-        snippet_path = match.group(1)
-
-        # pymdownx logic: try relative to current file, then relative to docs, then project root
-        candidates = [
-            base_path.parent / snippet_path,  # Relative to the doc file
-            docs_root / snippet_path,  # Relative to docs root
-            project_root / snippet_path,  # Relative to project root
-        ]
-
-        for candidate in candidates:
-            if candidate.exists() and candidate.is_file():
-                # Recursively resolve snippets inside the included file
-                return resolve_snippets(
-                    candidate.read_text(encoding="utf-8"), candidate
+    docs_path = _docs_resource.get_path()
+    examples_path = _examples_resource.get_path()
+
+    # Create schema with fields for title, path, and content
+    schema_builder = tantivy.SchemaBuilder()
+    schema_builder.add_text_field("title", stored=True)
+    schema_builder.add_text_field("path", stored=True)
+    schema_builder.add_text_field("content", stored=True)
+    schema_builder.add_text_field("type", stored=True)
+    schema = schema_builder.build()
+
+    # Create index in temporary directory
+    index = tantivy.Index(schema, path=tempfile.mkdtemp())
+    writer = index.writer()
+
+    # Helper to index files
+    def index_files(
+        root_path: Path,
+        pattern: str,
+        type_label: str,
+        path_prefix: str,
+        process_content=None,
+        extract_title=None,
+    ):
+        for file_path in root_path.rglob(pattern):
+            content = file_path.read_text(encoding="utf-8")
+            if process_content:
+                content = process_content(content, file_path)
+
+            rel_path = str(file_path.relative_to(root_path))
+            title = (
+                extract_title(content, file_path)
+                if extract_title
+                else file_path.stem
+            )
+
+            writer.add_document(
+                tantivy.Document(
+                    title=title,
+                    path=f"{path_prefix}/{rel_path}",
+                    content=content,
+                    type=type_label,
                 )
+            )
 
-        return f"> [!WARNING] Could not resolve snippet: {snippet_path}"
+    # Extract title from markdown first heading
+    def extract_md_title(content: str, file_path: Path) -> str:
+        for line in content.split("\n"):
+            if line.startswith("# "):
+                return line[2:].strip()
+        return file_path.stem
+
+    # For snippet resolution in search indexing
+    def resolve_for_indexing(content: str, file_path: Path) -> str:
+        from qtype.base.resources import _resolve_snippets
+
+        return _resolve_snippets(content, file_path, _docs_resource)
+
+    # Index documentation and examples
+    index_files(
+        docs_path,
+        "*.md",
+        "documentation",
+        "docs",
+        process_content=resolve_for_indexing,
+        extract_title=extract_md_title,
+    )
+    index_files(examples_path, "*.yaml", "example", "examples")
 
-    return SNIPPET_REGEX.sub(replace_match, content)
+    writer.commit()
+    return index
 
 
 # ============================================================================
@@ -122,6 +157,10 @@ class MermaidVisualizationResult(BaseModel):
     preview_instructions: str
 
 
+# Rebuild model after nested dependency is defined
+MermaidVisualizationResult.model_rebuild()
+
+
 @mcp.tool(
     title="Convert API Specification to QType Tools",
     description=(
@@ -287,36 +326,13 @@ def get_documentation(file_path: str) -> str:
             Use list_documentation to see all available files.
 
     Returns:
-        The full markdown content of the documentation file.
+        The full markdown content of the documentation file with snippets resolved.
 
     Raises:
         FileNotFoundError: If the specified file doesn't exist.
         ValueError: If the path tries to access files outside the docs directory.
     """
-    docs_path = _get_docs_path()
-
-    # Resolve the requested file path
-    requested_file = (docs_path / file_path).resolve()
-
-    # Security check: ensure the resolved path is within docs directory
-    try:
-        requested_file.relative_to(docs_path.resolve())
-    except ValueError:
-        raise ValueError(
-            f"Invalid path: '{file_path}' is outside documentation directory"
-        )
-
-    if not requested_file.exists():
-        raise FileNotFoundError(
-            f"Documentation file not found: '{file_path}'. "
-            "Use list_documentation to see available files."
-        )
-
-    if not requested_file.is_file():
-        raise ValueError(f"Path is not a file: '{file_path}'")
-
-    content = requested_file.read_text(encoding="utf-8")
-    return resolve_snippets(content, requested_file)
+    return _docs_resource.get_file(file_path)
 
 
 @mcp.tool(
@@ -363,21 +379,113 @@ def list_documentation() -> list[str]:
         Paths are relative to the docs root (e.g., "components/Flow.md",
         "Tutorials/getting_started.md").
     """
-    docs_path = _get_docs_path()
+    return _docs_resource.list_files()
 
-    if not docs_path.exists():
-        raise FileNotFoundError(
-            f"Documentation directory not found: {docs_path}"
-        )
 
-    # Find all markdown files
-    md_files = []
-    for md_file in docs_path.rglob("*.md"):
-        # Get relative path from docs root
-        rel_path = md_file.relative_to(docs_path)
-        md_files.append(str(rel_path))
+@mcp.tool(
+    title="Get QType Example",
+    description=(
+        "Returns the content of a specific example YAML file. "
+        "Use list_examples first to see available files. "
+        "Provide the relative path (e.g., 'conversational_ai/simple_chatbot.qtype.yaml', "
+        "'data_processing/csv_processor.qtype.yaml')."
+    ),
+)
+def get_example(file_path: str) -> str:
+    """Get the content of a specific example file.
+
+    Args:
+        file_path: Relative path to the example file from the examples root.
+            Example: "conversational_ai/simple_chatbot.qtype.yaml",
+            "data_processing/csv_processor.qtype.yaml".
+            Use list_examples to see all available files.
+
+    Returns:
+        The full YAML content of the example file.
+
+    Raises:
+        FileNotFoundError: If the specified file doesn't exist.
+        ValueError: If the path tries to access files outside the examples directory.
+    """
+    return _examples_resource.get_file(file_path)
+
+
+@mcp.tool(
+    title="List QType Examples",
+    description=(
+        "Returns a list of all available example YAML files. "
+        "Use this to discover what examples exist, then retrieve "
+        "specific files with get_example. Examples are organized by category: "
+        "conversational_ai/ (chatbots and agents), data_processing/ (ETL and transformations), "
+        "invoke_models/ (LLM usage), language_features/ (QType syntax examples), etc."
+    ),
+    structured_output=True,
+)
+def list_examples() -> list[str]:
+    """List all available example YAML files.
+
+    Returns:
+        Sorted list of relative paths to all .yaml example files.
+        Paths are relative to the examples root (e.g.,
+        "conversational_ai/simple_chatbot.qtype.yaml",
+        "data_processing/csv_processor.qtype.yaml").
+    """
+    return _examples_resource.list_files()
+
+
+@mcp.tool(
+    title="Search QType Library",
+    description=(
+        "Full-text search across all QType documentation and examples. "
+        "Returns matching documents and example YAML files ranked by relevance. "
+        "Use this to find documentation about specific topics, features, or components, "
+        "or to discover example implementations. Doc paths can be used with get_documentation."
+    ),
+    structured_output=True,
+)
+def search_library(query: str, limit: int = 10) -> list[dict[str, Any]]:
+    """Search library using full-text search.
+
+    Args:
+        query: Search query string. Can include multiple words, phrases,
+            or boolean operators (AND, OR, NOT).
+        limit: Maximum number of results to return (default: 10, max: 50).
+
+    Returns:
+        List of matching items with:
+        - title: Item title
+        - path: Relative path (docs/ or examples/ prefix)
+        - type: Either "documentation" or "example"
+        - score: Relevance score (higher is more relevant)
+
+    Examples:
+        search_library("flow execution")  # Find docs/examples about flows
+        search_library("DocumentSource")  # Find component docs
+        search_library("authentication AND API")  # Boolean search
+    """
+    # Clamp limit to reasonable range
+    limit = max(1, min(limit, 50))
+
+    index = _build_search_index()
+    index.reload()
+    searcher = index.searcher()
+    tantivy_query = index.parse_query(query, ["title", "content"])
+
+    search_results = searcher.search(tantivy_query, limit)
+
+    results = []
+    for score, doc_address in search_results.hits:
+        doc = searcher.doc(doc_address)
+        results.append(
+            {
+                "title": doc["title"][0],
+                "path": doc["path"][0],
+                "type": doc["type"][0],
+                "score": score,
+            }
+        )
 
-    return sorted(md_files)
+    return results
 
 
 @mcp.tool(

{schema → qtype/schema}/qtype.schema.json

@@ -68,20 +68,20 @@
           "type": "string"
         },
         "inputs": {
-          "additionalProperties": {
-            "$ref": "#/$defs/ToolParameter"
-          },
           "description": "Input parameters required by this tool.",
+          "items": {
+            "$ref": "#/$defs/Variable"
+          },
           "title": "Inputs",
-          "type": "object"
+          "type": "array"
         },
         "outputs": {
-          "additionalProperties": {
-            "$ref": "#/$defs/ToolParameter"
-          },
           "description": "Output parameters produced by this tool.",
+          "items": {
+            "$ref": "#/$defs/Variable"
+          },
           "title": "Outputs",
-          "type": "object"
+          "type": "array"
         },
         "type": {
           "const": "APITool",
@@ -125,12 +125,12 @@
           "type": "object"
         },
         "parameters": {
-          "additionalProperties": {
-            "$ref": "#/$defs/ToolParameter"
+          "description": "Path and query parameters for the API call.",
+          "items": {
+            "$ref": "#/$defs/Variable"
           },
-          "description": "Output parameters produced by this tool.",
           "title": "Parameters",
-          "type": "object"
+          "type": "array"
         }
       },
       "required": [
@@ -439,7 +439,7 @@
     },
     "Aggregate": {
       "additionalProperties": false,
-      "description": "A terminal step that consumes an entire input stream and produces a single\nsummary message with success/error counts.",
+      "description": "A step that, after all messages have been processed,\nreturns a single message containing the counts of successful and failed\nmessages. Other messages are passed through unchanged.",
       "properties": {
         "cache_config": {
           "anyOf": [
@@ -480,7 +480,7 @@
           "type": "array"
         },
         "outputs": {
-          "description": "References to the variables for the output. There should be one and only one output with type AggregateStats",
+          "description": "References to the variables where output is stored.",
           "items": {
             "anyOf": [
               {
@@ -1084,18 +1084,25 @@
           "title": "Outputs",
           "type": "array"
         },
-        "field_mapping": {
+        "field_bindings": {
           "additionalProperties": {
-            "type": "string"
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Reference_Variable_"
+              },
+              {
+                "type": "string"
+              }
+            ]
           },
-          "description": "Mapping of type inputs to variable names, if needed.",
-          "title": "Field Mapping",
+          "description": "Mapping from type field names to flow variable names.",
+          "title": "Field Bindings",
           "type": "object"
         }
       },
       "required": [
         "id",
-        "field_mapping"
+        "field_bindings"
       ],
       "title": "Construct",
       "type": "object"
@@ -1962,7 +1969,7 @@
     },
     "FieldExtractor": {
       "additionalProperties": false,
-      "description": "Extracts specific fields from input data using JSONPath expressions.\n\nThis step uses JSONPath syntax to extract data from structured inputs\n(Pydantic models, dicts, lists). The input is first converted to a dict\nusing model_dump() if it's a Pydantic model, then the JSONPath expression\nis evaluated.\n\nIf the JSONPath matches multiple values, the step yields multiple output\nmessages (1-to-many cardinality). If it matches a single value, it yields\none output message. If it matches nothing, it raises an error.\n\nThe extracted data is used to construct the output variable by passing it\nas keyword arguments to the output type's constructor.\n\nExample JSONPath expressions:\n- `$.field_name` - Extract a single field\n- `$.items[*]` - Extract all items from a list\n- `$.items[?(@.price > 10)]` - Filter items by condition",
+      "description": "Extracts specific fields from input data using JSONPath expressions.\n\nThis step uses JSONPath syntax to extract data from structured inputs\n(Pydantic models, dicts, lists). The input is first converted to a dict\nusing model_dump() if it's a Pydantic model, then the JSONPath expression\nis evaluated.\n\nIf the JSONPath matches multiple values, the step yields multiple output\nmessages (1-to-many cardinality). If it matches a single value, it yields\none output message. If it matches nothing, it raises an error.\n\nThe extracted data is used to construct the output variable by passing it\nas keyword arguments to the output type's constructor.\n\nIf there is no match and the output variable is optional, it is set to None.\nIf there is no match and the output variable is required, an error is raised.\n\nExample JSONPath expressions:\n- `$.field_name` - Extract a single field\n- `$.items[*]` - Extract all items from a list\n- `$.items[?(@.price > 10)]` - Filter items by condition",
       "properties": {
         "cache_config": {
           "anyOf": [
@@ -2021,12 +2028,6 @@
           "description": "JSONPath expression to extract data from the input. Uses jsonpath-ng syntax.",
           "title": "Json Path",
           "type": "string"
-        },
-        "fail_on_missing": {
-          "default": true,
-          "description": "Whether to raise an error if the JSONPath matches no data. If False, returns None.",
-          "title": "Fail On Missing",
-          "type": "boolean"
         }
       },
       "required": [
@@ -2678,23 +2679,31 @@
         },
         "input_bindings": {
           "additionalProperties": {
-            "type": "string"
-          },
-          "description": "Mapping from variable references to flow input variable IDs.",
-          "propertyNames": {
-            "$ref": "#/$defs/Reference_Variable_"
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Reference_Variable_"
+              },
+              {
+                "type": "string"
+              }
+            ]
           },
+          "description": "Mapping from flow input variable IDs to step variable names.",
           "title": "Input Bindings",
           "type": "object"
         },
         "output_bindings": {
           "additionalProperties": {
-            "type": "string"
-          },
-          "description": "Mapping from variable references to flow output variable IDs.",
-          "propertyNames": {
-            "$ref": "#/$defs/Reference_Variable_"
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Reference_Variable_"
+              },
+              {
+                "type": "string"
+              }
+            ]
           },
+          "description": "Mapping from flow output variable IDs to step variable names.",
           "title": "Output Bindings",
           "type": "object"
         }
@@ -2783,17 +2792,31 @@
         },
         "input_bindings": {
           "additionalProperties": {
-            "type": "string"
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Reference_Variable_"
+              },
+              {
+                "type": "string"
+              }
+            ]
           },
-          "description": "Mapping from variable references to tool input parameter names.",
+          "description": "Mapping from tool parameter names to flow variable names.",
           "title": "Input Bindings",
           "type": "object"
         },
         "output_bindings": {
           "additionalProperties": {
-            "type": "string"
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Reference_Variable_"
+              },
+              {
+                "type": "string"
+              }
+            ]
           },
-          "description": "Mapping from variable references to tool output parameter names.",
+          "description": "Mapping from tool output names to flow variable names.",
           "title": "Output Bindings",
           "type": "object"
         }
@@ -3230,20 +3253,20 @@
           "type": "string"
         },
         "inputs": {
-          "additionalProperties": {
-            "$ref": "#/$defs/ToolParameter"
-          },
           "description": "Input parameters required by this tool.",
+          "items": {
+            "$ref": "#/$defs/Variable"
+          },
           "title": "Inputs",
-          "type": "object"
+          "type": "array"
         },
         "outputs": {
-          "additionalProperties": {
-            "$ref": "#/$defs/ToolParameter"
-          },
           "description": "Output parameters produced by this tool.",
+          "items": {
+            "$ref": "#/$defs/Variable"
+          },
           "title": "Outputs",
-          "type": "object"
+          "type": "array"
         },
         "type": {
           "const": "PythonFunctionTool",
@@ -3658,37 +3681,6 @@
       "title": "ToolList",
       "type": "array"
     },
-    "ToolParameter": {
-      "description": "Defines a tool input or output parameter with type and optional flag.",
-      "properties": {
-        "type": {
-          "anyOf": [
-            {
-              "$ref": "#/$defs/PrimitiveTypeEnum"
-            },
-            {},
-            {
-              "$ref": "#/$defs/ListType"
-            },
-            {
-              "type": "string"
-            }
-          ],
-          "title": "Type"
-        },
-        "optional": {
-          "default": false,
-          "description": "Whether this parameter is optional",
-          "title": "Optional",
-          "type": "boolean"
-        }
-      },
-      "required": [
-        "type"
-      ],
-      "title": "ToolParameter",
-      "type": "object"
-    },
     "TypeList": {
       "description": "Schema for a standalone list of type definitions.",
       "items": {
@@ -3722,6 +3714,12 @@
           "description": "Type of data expected or produced. Either a CustomType or domain specific type.",
           "title": "Type"
         },
+        "optional": {
+          "default": false,
+          "description": "Whether this variable can be unset or None. Use '?' suffix in type string as shorthand (e.g., 'text?').",
+          "title": "Optional",
+          "type": "boolean"
+        },
         "ui": {
           "anyOf": [
             {

qtype/semantic/checker.py

@@ -50,6 +50,10 @@ class FlowHasNoStepsError(QTypeValidationError):
 # Alias for backward compatibility and semantic clarity
 QTypeSemanticError = SemanticError
 
+# Step types that support text streaming
+# These are the only step types that can produce streaming text output
+STREAMING_STEP_TYPES = (LLMInference, Agent)
+
 
 # ---- Helper Functions for Common Validation Patterns ----
 
@@ -543,6 +547,21 @@ def _validate_flow(flow: Flow) -> None:
                     f"Flow {flow.id} has a Complete interface but {len(text_outputs)} text outputs -- there should be 1."
                 )
 
+            # Ensure the final step supports streaming for Complete interface
+            if flow.steps:
+                final_step = flow.steps[-1]
+                if not isinstance(final_step, STREAMING_STEP_TYPES):
+                    streaming_type_names = ", ".join(
+                        t.__name__ for t in STREAMING_STEP_TYPES
+                    )
+                    raise QTypeSemanticError(
+                        (
+                            f"Flow {flow.id} has a Complete interface which requires streaming output, "
+                            f"but the final step '{final_step.id}' is of type '{final_step.type}' which does not support streaming. "
+                            f"The final step must be one of: {streaming_type_names}."
+                        )
+                    )
+
 
 def _has_secret_reference(obj: Any) -> bool:
     """