qtype 0.1.12__py3-none-any.whl → 0.1.13__py3-none-any.whl

qtype/commands/run.py

@@ -11,11 +11,10 @@ import warnings
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
 from pydantic.warnings import UnsupportedFieldAttributeWarning
 
-from qtype.application.facade import QTypeFacade
 from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
+from qtype.interpreter.converters import read_dataframe_from_file
 
 logger = logging.getLogger(__name__)
 
@@ -29,50 +28,124 @@ for name in ["httpx", "urllib3", "qdrant_client", "opensearch"]:
     logging.getLogger(name).setLevel(logging.WARNING)
 
 
-def read_data_from_file(file_path: str) -> pd.DataFrame:
-    """
-    Reads a file into a pandas DataFrame based on its MIME type.
+def register_telemetry(spec) -> None:
+    """Register telemetry if enabled in the spec."""
+    from qtype.interpreter.telemetry import register
+    from qtype.semantic.model import Application as SemanticApplication
+
+    if isinstance(spec, SemanticApplication) and spec.telemetry:
+        logger.info(
+            f"Telemetry enabled with endpoint: {spec.telemetry.endpoint}"
+        )
+        secret_mgr = create_secret_manager_for_spec(spec)
+        register(spec.telemetry, secret_mgr, spec.id)
+
+
+def create_secret_manager_for_spec(spec):
+    """Create a secret manager based on the specification."""
+    from qtype.interpreter.base.secrets import create_secret_manager
+    from qtype.semantic.model import Application as SemanticApplication
+
+    if isinstance(spec, SemanticApplication):
+        return create_secret_manager(spec.secret_manager)
+    else:
+        raise ValueError(
+            "Can't create secret manager for non-Application spec"
+        )
+
+
+async def execute_workflow(
+    path: Path,
+    inputs: dict | Any,
+    flow_name: str | None = None,
+    **kwargs: Any,
+) -> Any:
+    """Execute a complete workflow from document to results.
+
+    Args:
+        path: Path to the QType specification file
+        inputs: Dictionary of input values or DataFrame for batch
+        flow_name: Optional name of flow to execute
+        **kwargs: Additional dependencies for execution
+
+    Returns:
+        DataFrame with results (one row per input)
     """
-    from pathlib import Path
-
-    import magic
-
-    mime_type = magic.Magic(mime=True).from_file(file_path)
-
-    if mime_type == "text/csv":
-        # TODO: Restore na values and convert to optional once we support them https://github.com/bazaarvoice/qtype/issues/101
-        df = pd.read_csv(file_path)
-        return df.fillna("")
-    elif mime_type == "text/plain":
-        # For text/plain, use file extension to determine format
-        file_ext = Path(file_path).suffix.lower()
-        if file_ext == ".csv":
-            # TODO: Restore na values and convert to optional once we support them https://github.com/bazaarvoice/qtype/issues/101
-            df = pd.read_csv(file_path)
-            return df.fillna("")
-        elif file_ext == ".json":
-            return pd.read_json(file_path)
+    import pandas as pd
+    from opentelemetry import trace
+
+    from qtype.interpreter.base.executor_context import ExecutorContext
+    from qtype.interpreter.converters import (
+        dataframe_to_flow_messages,
+        flow_messages_to_dataframe,
+    )
+    from qtype.interpreter.flow import run_flow
+    from qtype.interpreter.types import Session
+    from qtype.semantic.loader import load
+    from qtype.semantic.model import Application as SemanticApplication
+
+    # Load the semantic application
+    semantic_model, type_registry = load(path)
+    assert isinstance(semantic_model, SemanticApplication)
+    register_telemetry(semantic_model)
+
+    # Find the flow to execute
+    if flow_name:
+        target_flow = None
+        for flow in semantic_model.flows:
+            if flow.id == flow_name:
+                target_flow = flow
+                break
+        if target_flow is None:
+            raise ValueError(f"Flow '{flow_name}' not found")
+    else:
+        if semantic_model.flows:
+            target_flow = semantic_model.flows[0]
         else:
-            raise ValueError(
-                (
-                    f"Unsupported text/plain file extension: {file_ext}. "
-                    "Supported extensions: .csv, .json"
-                )
-            )
-    elif mime_type == "application/json":
-        return pd.read_json(file_path)
-    elif mime_type in [
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-excel",
-    ]:
-        return pd.read_excel(file_path)
-    elif mime_type in ["application/vnd.parquet", "application/octet-stream"]:
-        return pd.read_parquet(file_path)
+            raise ValueError("No flows found in application")
+
+    logger.info(f"Executing flow {target_flow.id} from {path}")
+
+    # Convert inputs to DataFrame (normalize single dict to 1-row DataFrame)
+    if isinstance(inputs, dict):
+        input_df = pd.DataFrame([inputs])
+    elif isinstance(inputs, pd.DataFrame):
+        input_df = inputs
     else:
         raise ValueError(
-            f"Unsupported MIME type for file {file_path}: {mime_type}"
+            f"Inputs must be dict or DataFrame, got {type(inputs)}"
         )
 
+    # Create session
+    session = Session(
+        session_id=kwargs.pop("session_id", "default"),
+        conversation_history=kwargs.pop("conversation_history", []),
+    )
+
+    # Convert DataFrame to FlowMessages with type conversion
+    initial_messages_list = dataframe_to_flow_messages(
+        input_df, target_flow.inputs, session=session
+    )
+
+    # Execute the flow
+    secret_manager = create_secret_manager_for_spec(semantic_model)
+
+    context = ExecutorContext(
+        secret_manager=secret_manager,
+        tracer=trace.get_tracer(__name__),
+    )
+    results = await run_flow(
+        target_flow,
+        initial_messages_list,
+        context=context,
+        **kwargs,
+    )
+
+    # Convert results back to DataFrame
+    results_df = flow_messages_to_dataframe(results, target_flow)
+
+    return results_df
+
 
 def run_flow(args: Any) -> None:
     """Run a QType YAML spec file by executing its flows.
@@ -82,7 +155,6 @@ def run_flow(args: Any) -> None:
     """
     import asyncio
 
-    facade = QTypeFacade()
     spec_path = Path(args.spec)
 
     try:
@@ -90,7 +162,7 @@ def run_flow(args: Any) -> None:
 
         if args.input_file:
             logger.info(f"Loading input data from file: {args.input_file}")
-            input: Any = read_data_from_file(args.input_file)
+            input: Any = read_dataframe_from_file(args.input_file)
         else:
             # Parse input JSON
             try:
@@ -99,9 +171,9 @@ def run_flow(args: Any) -> None:
                 logger.error(f"❌ Invalid JSON input: {e}")
                 return
 
-        # Execute the workflow using the facade (now async, returns DataFrame)
+        # Execute the workflow using the standalone function
         result_df = asyncio.run(
-            facade.execute_workflow(
+            execute_workflow(
                 spec_path,
                 flow_name=args.flow,
                 inputs=input,

qtype/docs/.pages

@@ -0,0 +1,8 @@
+nav:
+  - index.md
+  - Tutorials
+  - Gallery
+  - How To
+  - Concepts
+  - Reference
+  - Contributing

{docs → qtype/docs}/Concepts/mental-model-and-philosophy.md

@@ -128,7 +128,7 @@ Output Variables
 
 **Linear execution:** Steps run sequentially in declaration order. Each step waits for its inputs to be available. Parallelism is supported for multiple inputs.
 
-**1-to-many cardinality:** Some steps (like `Explode`) can produce multiple outputs for one input, creating fan-out patterns. Other steps (like `Collect`) aggregate many inputs into one output. This enables batch processing patterns.
+**1-to-many cardinality:** Some steps (like `Explode`) can produce multiple outputs for one input, creating fan-out patterns. Other steps (like `Collect`) gather many inputs into one collection. This enables batch processing patterns.
 
 ---
 

qtype/docs/Contributing/.pages

@@ -0,0 +1,4 @@
+title: Contributing
+nav:
+  - index.md
+  - roadmap.md

{docs → qtype/docs}/Contributing/index.md

@@ -57,6 +57,14 @@ After installation, you should be able to run the `qtype` command from anywhere:
 qtype --help
 ```
 
+## Running the MCP In Dev Mode
+
+To start it in `dev` mode with the inspector:
+```
+mcp dev qtype/mcp/server.py:mcp
+```
+
+
 ## Running Tests
 
 The project uses pytest for testing with coverage measurement:
@@ -161,7 +169,6 @@ pre-commit install
 
 Settings are in `.pre-commit-config.yaml`:
 
-
 ##  Project Structure
 
 - `qtype/` – Python package for parsing, validating, and interpreting QType specs

{docs → qtype/docs}/Gallery/dataflow_pipelines.md

@@ -76,5 +76,6 @@ result_file: results.parquet
 
 ## Learn More
 
-- Tutorial: [Your First QType Application](../../Tutorials/01_hello_world.md)
-- Example: [Simple Chatbot](./simple_chatbot.md)
+- [Tutorial: Your First QType Application](../Tutorials/01-first-qtype-application.md)
+- [Read Data from SQL Databases](../How%20To/Data%20Processing/read_sql_databases.md)
+- [Adjust Concurrency](../How%20To/Data%20Processing/adjust_concurrency.md)

{docs → qtype/docs}/Gallery/research_assistant.md

@@ -92,7 +92,6 @@ When running with the topic "Latest developments in retrieval augmented generati
 
 ## Learn More
 
-- How-To: [Create Tools from OpenAPI Specifications](../How%20To/Tools%20%26%20Integration/create_tools_from_openapi_specifications.md)
-- How-To: [Bind Tool Inputs and Outputs](../How%20To/Tools%20%26%20Integration/bind_tool_inputs_and_outputs.md)
-- How-To: [Include QType YAML](../How%20To/Language%20Features/include_qtype_yaml.md)
-- How-To: [Call Large Language Models](../How%20To/Invoke%20Models/call_large_language_models.md)
+- [Create Tools from OpenAPI Specifications](../How%20To/Tools%20%26%20Integration/create_tools_from_openapi_specifications.md)
+- [Bind Tool Inputs and Outputs](../How%20To/Tools%20%26%20Integration/bind_tool_inputs_and_outputs.md)
+- [Call Large Language Models](../How%20To/Invoke%20Models/call_large_language_models.md)

{docs → qtype/docs}/Gallery/simple_chatbot.md

@@ -33,4 +33,6 @@ qtype serve examples/conversational_ai/simple_chatbot.qtype.yaml
 
 ## Learn More
 
-- Tutorial: [Building a Stateful Chatbot](../../Tutorials/02_conversational_chat.md)
+- [Tutorial: Conversational Chatbot](../Tutorials/02-conversational-chatbot.md)
+- [Use Conversational Interfaces](../How%20To/Qtype%20Server/use_conversational_interfaces.md)
+- [ChatMessage Reference](../components/ChatMessage.md)

{docs → qtype/docs}/How To/Authentication/configure_aws_authentication.md

@@ -55,6 +55,6 @@ models:
 ## See Also
 
 - [AWSAuthProvider Reference](../../components/AWSAuthProvider.md)
+- [Use API Key Authentication](use_api_key_authentication.md)
+- [Call Large Language Models](../Invoke%20Models/call_large_language_models.md)
 - [Model Reference](../../components/Model.md)
-- [How-To: Use API Key Authentication](use_api_key_authentication.md)
-- [How-To: Manage Secrets with Secret Manager](../Authentication/manage_secrets.md)

{docs → qtype/docs}/How To/Authentication/use_api_key_authentication.md

@@ -36,5 +36,5 @@ models:
 
 - [APIKeyAuthProvider Reference](../../components/APIKeyAuthProvider.md)
 - [Use Environment Variables](../Language%20Features/use_environment_variables.md)
-- [Model Reference](../../components/Model.md)
-- [Tutorial: Your First QType Application](../../Tutorials/your_first_qtype_application.md)
+- [Configure AWS Authentication](configure_aws_authentication.md)
+- [Tutorial: Your First QType Application](../../Tutorials/01-first-qtype-application.md)

{docs → qtype/docs}/How To/Command Line Usage/load_multiple_inputs_from_files.md

@@ -10,8 +10,8 @@ qtype run app.qtype.yaml --input-file inputs.csv
 
 ### Supported File Formats
 
-- **CSV**: Columns map to input variable names
-- **JSON**: Array of objects or records format
+- **CSV**: Columns map to input variable names (best for primitive types)
+- **JSON**: Array of objects or records format (best for nested/complex types)
 - **Parquet**: Efficient columnar format for large datasets
 - **Excel**: `.xlsx` or `.xls` files
 
@@ -19,10 +19,25 @@ qtype run app.qtype.yaml --input-file inputs.csv
 
 When you provide `--input-file`, QType:
 1. Reads the file into a pandas DataFrame
-2. Each row becomes one execution of the flow
-3. Column names must match flow input variable IDs
-4. Processes rows with configured concurrency
-5. Returns results as a DataFrame (can be saved with `--output`)
+2. Automatically converts data to match input variable types
+3. Each row becomes one execution of the flow
+4. Column names must match flow input variable IDs
+5. Processes rows with configured concurrency
+6. Returns results as a DataFrame (can be saved with `--output`)
+
+### Type Conversion
+
+QType automatically converts file data to match your flow's input types:
+
+- **Primitive types** (`int`, `float`, `bool`, `text`): Converted from file values
+- **Custom types**: Validated and instantiated from dict/object columns (use JSON format)
+- **Domain types**: Built-in types like `ChatMessage` or `SearchResult` (use JSON format)
+
+**Format Selection Guide:**
+
+- Use **CSV** for simple data with primitive types (strings, numbers, booleans)
+- Use **JSON** for complex data with custom types, nested objects, or domain types
+- Use **Parquet** for large datasets with mixed types and efficient storage
 
 ## Complete Example
 
@@ -57,6 +72,6 @@ qtype run batch_processing.qtype.yaml \
 
 ## See Also
 
-<!-- - [Adjust Concurrency](adjust_concurrency.md) -->
-<!-- - [FileSource Reference](../../components/FileSource.md) -->
-- [Example: Dataflow Pipeline](../../Gallery/Data%20Processing/dataflow_pipelines.md)
+- [Pass Inputs On The CLI](pass_inputs_on_the_cli.md)
+- [Adjust Concurrency](../Data%20Processing/adjust_concurrency.md)
+- [Gallery: Dataflow Pipelines](../../Gallery/dataflow_pipelines.md)

{docs → qtype/docs}/How To/Command Line Usage/pass_inputs_on_the_cli.md

@@ -47,6 +47,6 @@ flows:
 
 ## See Also
 
-- [Load Multiple Inputs from Files](load_inputs_from_files.md)
-- [Use Session Inputs for Sticky Variables](../Language%20Features/use_session_inputs.md)
-- [Example: LLM Processing Pipelines](../../Gallery/dataflow_pipelines.md)
+- [Load Multiple Inputs from Files](load_multiple_inputs_from_files.md)
+- [CLI Reference](../../Reference/cli.md)
+- [Gallery: Dataflow Pipelines](../../Gallery/dataflow_pipelines.md)

{docs → qtype/docs}/How To/Command Line Usage/serve_with_auto_reload.md

@@ -22,5 +22,6 @@ qtype serve --reload -p 8080 examples/tutorials/01_hello_world.qtype.yaml
 
 ## See Also
 
-- [Serve Command Reference](../../Reference/CLI.md#serve)
-- [Tutorial: Hello World](../../Tutorials/01_hello_world.md)
+- [Serve Flows as APIs](../Qtype%20Server/serve_flows_as_apis.md)
+- [Serve Flows as UI](../Qtype%20Server/serve_flows_as_ui.md)
+- [CLI Reference](../../Reference/cli.md)

{docs → qtype/docs}/How To/Data Processing/adjust_concurrency.md

@@ -35,7 +35,6 @@ The following step types support `concurrency_config`:
 
 ## See Also
 
-- [LLMInference Reference](../../components/LLMInference.md)
-- [InvokeEmbedding Reference](../../components/InvokeEmbedding.md)
-- [DocumentEmbedder Reference](../../components/DocumentEmbedder.md)
-- [LLM Processing Pipelines](../../Gallery/dataflow_pipelines.md)
+- [Step Reference](../../components/Step.md)
+- [Cache Step Results](cache_step_results.md)
+- [Gallery: Dataflow Pipelines](../../Gallery/dataflow_pipelines.md)

{docs → qtype/docs}/How To/Data Processing/cache_step_results.md

@@ -66,6 +66,6 @@ qtype run examples/data_processing/cache_step_results.qtype.yaml  --progress -i
 
 ## See Also
 
-- [LLMInference Reference](../../components/LLMInference.md)
+- [Step Reference](../../components/Step.md)
 - [Adjust Concurrency](adjust_concurrency.md)
-- [Tutorial: Your First QType Application](../../Tutorials/your_first_qtype_application.md)
+- [Tutorial: Your First QType Application](../../Tutorials/01-first-qtype-application.md)

{docs → qtype/docs}/How To/Data Processing/decode_json_xml.md

@@ -21,4 +21,4 @@ Parse string data in JSON or XML format into structured outputs. This is particu
 
 - [Decoder Reference](../../components/Decoder.md)
 - [CustomType Reference](../../components/CustomType.md)
-- [Tutorial: Working with Types and Structured Data](../../Tutorials/structured_data.md)
+- [Tutorial: Structured Data](../../Tutorials/03-structured-data.md)

{docs → qtype/docs}/How To/Data Processing/explode_collections.md

@@ -35,6 +35,6 @@ qtype run examples/data_processing/explode_items.qtype.yaml \
 
 ## See Also
 
-- [Aggregate Data using Collect](./aggregate_data.md)
+- [Gather Results into a List](gather_results.md)
 - [Explode Reference](../../components/Explode.md)
-- [Adjust Concurrency](./adjust_concurrency.md)
+- [Adjust Concurrency](adjust_concurrency.md)

{docs → qtype/docs}/How To/Data Processing/gather_results.md

@@ -22,8 +22,8 @@ steps:
 
 - **Collect**: Gathers all input values from multiple messages into a single list output
 - **Common ancestors**: Only variables that have the exact same value across ALL input messages are preserved in the output message
-- **Fan-out pattern**: Typically used after `Explode` to reverse the fan-out and aggregate results
-- **Single output**: Always produces exactly one output message containing the aggregated list
+- **Fan-out pattern**: Typically used after `Explode` to reverse the fan-out and accumulate results
+- **Single output**: Always produces exactly one output message containing the accumulate list
 
 ### Understanding Common Ancestors
 
@@ -63,6 +63,6 @@ all_processed: ['Processed: Phone', 'Processed: Laptop', 'Processed: Tablet']
 
 ## See Also
 
-- [Explode Collections for Fan-Out Processing](explode_collections.md)
+- [Explode Collections](explode_collections.md)
 - [Collect Reference](../../components/Collect.md)
-- [Explode Reference](../../components/Explode.md)
+- [Aggregate Reference](../../components/Aggregate.md)

qtype/docs/How To/Data Processing/invoke_other_flows.md

@@ -0,0 +1,71 @@
+# Invoke Other Flows
+
+Reuse flows as composable building blocks by invoking them from other flows with input and output bindings.
+
+### QType YAML
+
+```yaml
+flows:
+  # Define reusable flow
+  - type: Flow
+    id: summarize_text
+    variables:
+      - id: input_text
+        type: text
+      - id: output_summary
+        type: text
+    inputs: [input_text]
+    outputs: [output_summary]
+    steps:
+      - type: LLMInference
+        id: summarizer
+        model: my_model
+        inputs: [input_text]
+        outputs: [output_summary]
+
+  # Main flow invokes the reusable flow
+  - type: Flow
+    id: main
+    variables:
+      - id: article
+        type: text
+      - id: summary
+        type: text
+    inputs: [article]
+    outputs: [summary]
+    steps:
+      - type: InvokeFlow
+        id: get_summary
+        flow: summarize_text           # Reference to flow by ID
+        input_bindings:
+          input_text: article          # Map flow input to step variable
+        output_bindings:
+          output_summary: summary      # Map flow output to step variable
+```
+
+### Explanation
+
+- **InvokeFlow**: Step type that executes another flow with variable mapping
+- **flow**: ID of the flow to invoke (must be defined in the application)
+- **input_bindings**: Maps flow input variables to the invoking step's variables (format: `flow_input_name: step_variable_name`)
+- **output_bindings**: Maps flow output variables to the invoking step's variables (format: `flow_output_name: step_variable_name`)
+- **Reusability**: Flows can be invoked multiple times with different bindings
+
+## Complete Example
+
+```yaml
+--8<-- "../examples/data_processing/invoke_other_flows.qtype.yaml"
+```
+
+**Run it:**
+```bash
+qtype run examples/data_processing/invoke_other_flows.qtype.yaml \
+  --flow main \
+  --input '{"article_text": "Your article text here..."}'
+```
+
+## See Also
+
+- [InvokeFlow Reference](../../components/InvokeFlow.md)
+- [Flow Reference](../../components/Flow.md)
+- [Use Agent Skills](../Language%20Features/use_agent_skills.md)

qtype/docs/How To/Data Processing/load_data_from_athena.md

@@ -0,0 +1,49 @@
+# Load Data from Athena
+
+Query AWS Athena databases using standard SQL with the `SQLSource` step, which supports Athena through SQLAlchemy connection strings and AWS authentication.
+
+### QType YAML
+
+```yaml
+flows:
+  - id: query-athena
+    steps:
+      - type: SQLSource
+        id: load_sales
+        connection: "awsathena+rest://:@athena.us-east-1.amazonaws.com:443/sales_db?s3_staging_dir=s3://my-results-bucket/athena-results/&work_group=primary&catalog_name=some_catalog"
+        query: |
+          SELECT
+            product_id,
+            product_name,
+            total_sales
+          FROM product_sales
+          WHERE total_sales >= :min_sales
+          ORDER BY total_sales DESC
+        inputs:
+          - min_sales
+        outputs:
+          - product_id
+          - product_name
+          - total_sales
+```
+
+### Explanation
+
+- **awsathena+rest**: PyAthena SQLAlchemy dialect for accessing Athena via REST API
+- **Connection string format**: `awsathena+rest://:@athena.{REGION}.amazonaws.com:443/{DATABASE}?s3_staging_dir={S3_PATH}&work_group={WORKGROUP}&catalog_name={CATALOG}"`
+- **s3_staging_dir**: S3 location where Athena writes query results (required by Athena)
+- **work_group**: Athena workgroup name (e.g., `primary`)
+- **auth**: Reference to AWSAuthProvider for AWS credentials
+- **query**: Standard SQL query with parameter substitution using `:parameter_name` syntax
+
+## Complete Example
+
+```yaml
+--8<-- "../examples/data_processing/athena_query.qtype.yaml"
+```
+
+## See Also
+
+- [SQLSource Reference](../../components/SQLSource.md)
+- [Configure AWS Authentication](../Authentication/configure_aws_authentication.md)
+- [Read Data from SQL Databases](read_sql_databases.md)

qtype/docs/How To/Data Processing/read_data_from_files.md

@@ -0,0 +1,61 @@
+# Read Data from Files
+
+Load structured data from files using FileSource, which supports CSV, JSON, JSONL, and Parquet formats with automatic format detection and type conversion.
+
+### QType YAML
+
+```yaml
+steps:
+  - id: read_data
+    type: FileSource
+    path: batch_inputs.csv
+    outputs:
+      - query
+      - topic
+```
+
+### Explanation
+
+- **FileSource**: Step that reads structured data from files using fsspec-compatible URIs
+- **path**: File path (relative to YAML file or absolute), supports local files and cloud storage (s3://, gs://, etc.)
+- **outputs**: Column names from the file to extract as variables (must match actual column names)
+- **Format detection**: Automatically determined by file extension (.csv, .json, .jsonl, .parquet)
+- **Type conversion**: Automatically converts data to match variable types (primitives, domain types, custom types)
+- **Streaming**: Emits one FlowMessage per row, enabling downstream steps to process data in parallel
+
+### Automatic Type Conversion
+
+FileSource automatically converts data from files to match your variable types:
+
+- **Primitive types** (`int`, `float`, `bool`, `text`): Direct conversion from file data
+- **Domain types** (`ChatMessage`, `SearchResult`, etc.): Validated from dict/object columns
+- **Custom types**: Your defined types are validated and instantiated from dict/object columns
+
+**Format Recommendations:**
+
+- **CSV**: Best for simple primitive types (strings, numbers, booleans)
+- **JSON/JSONL**: Recommended for nested objects, custom types, and domain types
+- **Parquet**: Best for large datasets with mixed types and efficient storage
+
+**Example with Custom Types (JSON format):**
+
+```json
+[
+  {"person": {"name": "Alice", "age": 30}, "score": 95},
+  {"person": {"name": "Bob", "age": 25}, "score": 87}
+]
+```
+
+JSON preserves nested objects, making it ideal for complex types. CSV stores everything as strings, requiring nested objects to be serialized as JSON strings within the CSV.
+
+## Complete Example
+
+```yaml
+--8<-- "../examples/data_processing/read_file.qtype.yaml"
+```
+
+## See Also
+
+- [FileSource Reference](../../components/FileSource.md)
+- [Load Multiple Inputs from Files](../Command%20Line%20Usage/load_multiple_inputs_from_files.md)
+- [Write Data to File](write_data_to_file.md)

{docs → qtype/docs}/How To/Data Processing/read_sql_databases.md

@@ -42,6 +42,5 @@ steps:
 ## See Also
 
 - [SQLSource Reference](../../components/SQLSource.md)
-- [FileSource Reference](../../components/FileSource.md)
-- [Tutorial: Working with Types and Structured Data](../../Tutorials/working_with_types_and_structured_data.md)
-- [Example: Dataflow Pipeline](../../Gallery/Data%20Processing/dataflow_pipelines.md)
+- [Load Data from Athena](load_data_from_athena.md)
+- [Read Data from Files](read_data_from_files.md)

{docs → qtype/docs}/How To/Data Processing/write_data_to_file.md

@@ -36,5 +36,4 @@ See the [LLM Processing Pipelines](../../Gallery/dataflow_pipelines.md) gallery
 
 - [FileWriter Reference](../../components/FileWriter.md)
 - [Read Data from Files](read_data_from_files.md)
-- [Read SQL Databases](read_sql_databases.md)
-- [LLM Processing Pipelines](../../Gallery/dataflow_pipelines.md)
+- [Gallery: Dataflow Pipelines](../../Gallery/dataflow_pipelines.md)

{docs → qtype/docs}/How To/Invoke Models/call_large_language_models.md

@@ -48,4 +48,4 @@ qtype run simple_llm_call.qtype.yaml --input '{"text": "What is the capital of F
 
 - [LLMInference Reference](../../components/LLMInference.md)
 - [Model Reference](../../components/Model.md)
-- [Tutorial: Build a Conversational Interface](../../Tutorials/conversational_interface.md)
+- [Tutorial: Conversational Chatbot](../../Tutorials/02-conversational-chatbot.md)

{docs → qtype/docs}/How To/Invoke Models/create_embeddings.md

@@ -46,4 +46,4 @@ qtype run examples/invoke_models/create_embeddings.qtype.yaml \
 
 - [InvokeEmbedding Reference](../../components/InvokeEmbedding.md)
 - [EmbeddingModel Reference](../../components/EmbeddingModel.md)
-- [Tutorial: Build a RAG System](../../Tutorials/building_rag_system.md)
+- [Embedding Reference](../../components/Embedding.md)