qtype 0.1.11__py3-none-any.whl → 0.1.13__py3-none-any.whl

qtype/examples/data_processing/athena_query.qtype.yaml

@@ -0,0 +1,56 @@
+id: athena-query-example
+description: Query AWS Athena database and process results
+
+auths:
+  - type: aws
+    id: aws_auth
+    region: us-east-1
+    profile_name: default
+
+flows:
+  - type: Flow
+    id: query-athena
+
+    variables:
+      - id: min_sales
+        type: int
+      - id: product_id
+        type: text
+      - id: product_name
+        type: text
+      - id: total_sales
+        type: int
+      - id: region
+        type: text
+
+    inputs:
+      - min_sales
+
+    outputs:
+      - product_id
+      - product_name
+      - total_sales
+      - region
+
+    steps:
+      - type: SQLSource
+        id: load_sales
+        connection: "awsathena+rest://:@athena.us-east-1.amazonaws.com:443/sales_db?s3_staging_dir=s3://my-results-bucket/athena-results/&work_group=primary&catalog_name=some_catalog""
+        auth: aws_auth
+        query: |
+          SELECT
+            product_id,
+            product_name,
+            total_sales,
+            region
+          FROM product_sales
+          WHERE total_sales >= :min_sales
+          ORDER BY total_sales DESC
+          LIMIT 100
+        inputs:
+          - min_sales
+        outputs:
+          - product_id
+          - product_name
+          - total_sales
+          - region

qtype/examples/data_processing/batch_inputs.csv

@@ -0,0 +1,5 @@
+query,topic
+What is machine learning?,Technology
+Best pasta recipes,Cooking
+How to train for a marathon,Fitness
+Climate change solutions,Environment

qtype/examples/data_processing/batch_processing.qtype.yaml

@@ -0,0 +1,54 @@
+id: batch_processing_example
+description: Process multiple inputs loaded from a file
+
+models:
+  - type: Model
+    id: nova_lite
+    provider: aws-bedrock
+    model_id: amazon.nova-lite-v1:0
+
+flows:
+  - type: Flow
+    id: process_queries
+    description: Process multiple queries in batch
+
+    variables:
+      - id: query
+        type: text
+      - id: topic
+        type: text
+      - id: prompt
+        type: text
+      - id: response
+        type: text
+
+    inputs:
+      - query
+      - topic
+
+    outputs:
+      - query
+      - topic
+      - response
+
+    steps:
+      - id: create_prompt
+        type: PromptTemplate
+        template: |
+          Category: {topic}
+          Question: {query}
+          
+          Provide a brief, helpful answer:
+        inputs:
+          - query
+          - topic
+        outputs:
+          - prompt
+
+      - id: answer_query
+        type: LLMInference
+        model: nova_lite
+        inputs:
+          - prompt
+        outputs:
+          - response

qtype/examples/data_processing/cache_step_results.qtype.yaml

@@ -0,0 +1,78 @@
+id: cache_step_demo
+description: |
+  Demonstrates step caching to avoid redundant computation.
+  On first run, classifications are cached. On subsequent runs with same inputs,
+  results are retrieved from cache.
+
+auths:
+  - type: aws
+    id: aws_auth
+    region: us-east-1
+
+models:
+  - type: Model
+    id: nova
+    provider: aws-bedrock
+    model_id: us.amazon.nova-micro-v1:0
+    auth: aws_auth
+
+flows:
+  - id: classify_documents
+    variables:
+      - id: file_path
+        type: text
+      - id: document
+        type: text
+      - id: prompt
+        type: text
+      - id: category
+        type: text
+      - id: output_file
+        type: text
+
+    inputs:
+      - file_path
+
+    outputs:
+      - output_file
+
+    steps:
+      - type: FileSource
+        id: load_docs
+        path: file_path
+        outputs: [document]
+
+      - type: PromptTemplate
+        id: create_classification_prompt
+        template: |
+          Classify this document into one of these categories:
+          - Technology
+          - Finance
+          - Healthcare
+          - Education
+          
+          Document: {document}
+          
+          Reply with only the category name.
+        inputs: [document]
+        outputs: [prompt]
+
+      - type: LLMInference
+        id: classify
+        model: nova
+        inputs: [prompt]
+        outputs: [category]
+        # Enable caching with configuration
+        cache_config:
+          namespace: document_classification  # Logical separation for cache
+          version: "1.0"                       # Bump this to invalidate cache
+          on_error: Drop                       # Don't cache errors (default)
+          ttl: 3600                            # Cache for 1 hour (seconds)
+          compress: false                      # Optionally compress cached data
+
+      - type: FileWriter
+        id: write_results
+        path:
+          uri: classification_results.parquet
+        inputs: [document, category]
+        outputs: [output_file]

qtype/examples/data_processing/collect_results.qtype.yaml

@@ -0,0 +1,55 @@
+id: collect_example
+description: Aggregate fan-out results preserving only common ancestor variables
+
+flows:
+  - type: Flow
+    id: main
+    description: Process products by category, then collect results
+
+    variables:
+      - id: category
+        type: text
+      - id: region
+        type: text
+      - id: products
+        type: list[text]
+      - id: product
+        type: text
+      - id: processed_product
+        type: text
+      - id: all_processed
+        type: list[text]
+
+    inputs:
+      - category
+      - region
+      - products
+
+    outputs:
+      - all_processed
+
+    steps:
+      # Explode creates multiple messages, each with:
+      # - category (same for all)
+      # - region (same for all)
+      # - product (different for each)
+      - type: Explode
+        id: fan_out
+        inputs: [products]
+        outputs: [product]
+
+      # Each message still has category, region, and its unique product
+      - type: PromptTemplate
+        id: process
+        inputs: [product]
+        outputs: [processed_product]
+        template: "Processed: {product}"
+
+      # Collect aggregates all processed_product values into a list
+      # Only category and region are preserved as "common ancestors"
+      # (same value across all messages)
+      # The unique product variable is NOT preserved (different in each)
+      - type: Collect
+        id: aggregate
+        inputs: [processed_product]
+        outputs: [all_processed]

qtype/examples/data_processing/create_sample_db.py

@@ -0,0 +1,129 @@
+"""Create a sample SQLite database with product reviews for the example."""
+
+import sqlite3
+from pathlib import Path
+
+# Sample product reviews data
+SAMPLE_REVIEWS = [
+    (
+        1,
+        "Wireless Headphones",
+        5,
+        "Amazing sound quality! The noise cancellation is superb and "
+        "battery lasts all day. Highly recommend for music lovers.",
+    ),
+    (
+        2,
+        "Wireless Headphones",
+        2,
+        "Disappointed with the build quality. They broke after just 2 "
+        "weeks of normal use. Sound is okay but not worth the price.",
+    ),
+    (
+        3,
+        "Smart Watch",
+        4,
+        "Great fitness tracker with accurate heart rate monitoring. "
+        "Battery life could be better, but overall very satisfied.",
+    ),
+    (
+        4,
+        "Smart Watch",
+        5,
+        "Best smartwatch I've owned! Seamless integration with my phone, "
+        "tons of useful features, and looks professional.",
+    ),
+    (
+        5,
+        "Laptop Stand",
+        3,
+        "Does the job but feels flimsy. The adjustability is limited and "
+        "it wobbles a bit. Expected better quality for the price.",
+    ),
+    (
+        6,
+        "Laptop Stand",
+        5,
+        "Perfect for my home office setup! Sturdy construction, multiple "
+        "height options, and really helps with posture.",
+    ),
+    (
+        7,
+        "USB-C Hub",
+        4,
+        "Works well with all my devices. All ports function properly and "
+        "data transfer is fast. Gets a bit warm during heavy use.",
+    ),
+    (
+        8,
+        "USB-C Hub",
+        1,
+        "Stopped working after a week. One port was DOA and then the whole "
+        "hub died. Total waste of money.",
+    ),
+    (
+        9,
+        "Mechanical Keyboard",
+        5,
+        "Typing feels incredible! The switches are responsive and the build "
+        "quality is excellent. Worth every penny.",
+    ),
+    (
+        10,
+        "Mechanical Keyboard",
+        4,
+        "Great keyboard for coding. Switches are a bit loud for an office "
+        "environment but the tactile feedback is amazing.",
+    ),
+]
+
+
+def create_database(db_path: Path | str) -> None:
+    """Create SQLite database with sample product reviews.
+
+    Args:
+        db_path: Path where the database file should be created
+    """
+    db_path = Path(db_path)
+
+    # Remove existing database if it exists
+    if db_path.exists():
+        db_path.unlink()
+
+    # Create database and table
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Create reviews table
+    cursor.execute(
+        """
+        CREATE TABLE product_reviews (
+            review_id INTEGER PRIMARY KEY,
+            product_name TEXT NOT NULL,
+            rating INTEGER NOT NULL CHECK (rating >= 1 AND rating <= 5),
+            review_text TEXT NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """
+    )
+
+    # Insert sample data
+    cursor.executemany(
+        """
+        INSERT INTO product_reviews
+        (review_id, product_name, rating, review_text)
+        VALUES (?, ?, ?, ?)
+    """,
+        SAMPLE_REVIEWS,
+    )
+
+    conn.commit()
+    conn.close()
+
+    print(f"Created database at {db_path} with {len(SAMPLE_REVIEWS)} reviews")
+
+
+if __name__ == "__main__":
+    # Create database in the data_processing examples directory
+    db_path = Path(__file__).parent / "reviews.db"
+    create_database(db_path)

qtype/examples/data_processing/dataflow_pipelines.qtype.yaml

@@ -0,0 +1,108 @@
+id: review_analysis_pipeline
+description: |
+  Automated product review analysis pipeline demonstrating dataflow processing.
+  Reads reviews from SQLite database, analyzes sentiment with LLM, and writes
+  enriched results to a Parquet file.
+
+models:
+  - type: Model
+    id: nova_lite
+    provider: aws-bedrock
+    model_id: amazon.nova-lite-v1:0
+    inference_params:
+      temperature: 0.7
+      max_tokens: 256
+
+flows:
+  - type: Flow
+    id: analyze_reviews
+    description: Batch process product reviews with LLM sentiment analysis
+
+    variables:
+      - id: review_id
+        type: int
+      - id: product_name
+        type: text
+      - id: rating
+        type: int
+      - id: review_text
+        type: text
+      - id: analysis_prompt
+        type: text
+      - id: llm_analysis
+        type: text
+      - id: output_path
+        type: text
+      - id: result_file
+        type: text
+
+    inputs:
+      - output_path
+
+    outputs:
+      - result_file
+
+    steps:
+      # Step 1: Read reviews from SQLite database
+      # SQLSource emits one message per database row
+      - id: load_reviews
+        type: SQLSource
+        connection: "sqlite:///examples/data_processing/reviews.db"
+        query: |
+          SELECT
+            review_id,
+            product_name,
+            rating,
+            review_text
+          FROM product_reviews
+          ORDER BY review_id
+        inputs: []
+        outputs:
+          - review_id
+          - product_name
+          - rating
+          - review_text
+
+      # Step 2: Format analysis prompt for each review
+      # PromptTemplate creates structured prompts from review data
+      - id: create_prompt
+        type: PromptTemplate
+        template: |
+          Analyze this product review in 1-2 sentences. Include:
+          - Overall sentiment (positive/negative/mixed)
+          - 2-3 key themes or points
+
+          Product: {product_name}
+          Rating: {rating}/5
+          Review: {review_text}
+        inputs:
+          - product_name
+          - rating
+          - review_text
+        outputs:
+          - analysis_prompt
+
+      # Step 3: Analyze each review with LLM
+      # LLMInference processes each message through the language model
+      - id: analyze_sentiment
+        type: LLMInference
+        model: nova_lite
+        inputs:
+          - analysis_prompt
+        outputs:
+          - llm_analysis
+
+      # Step 4: Write enriched results to Parquet file
+      # FileWriter batches all messages and writes once
+      - id: write_results
+        type: FileWriter
+        path: output_path
+        inputs:
+          - review_id
+          - product_name
+          - rating
+          - review_text
+          - llm_analysis
+          - output_path
+        outputs:
+          - result_file

qtype/examples/data_processing/decode_json.qtype.yaml

@@ -0,0 +1,23 @@
+id: decode_json_example
+description: Decode JSON string into structured data
+
+flows:
+  - id: decode_product
+    description: Parse JSON string into variables
+    inputs: [json_string]
+    outputs: [name, price]
+    
+    variables:
+      - id: json_string
+        type: text
+      - id: name
+        type: text
+      - id: price
+        type: float
+    
+    steps:
+      - type: Decoder
+        id: parse_json
+        format: json
+        inputs: [json_string]
+        outputs: [name, price]

qtype/examples/data_processing/explode_items.qtype.yaml

@@ -0,0 +1,25 @@
+id: explode_example
+description: Explode a list into individual items for fan-out processing
+
+flows:
+  - type: Flow
+    id: main
+    description: Takes a list and processes each item individually
+
+    variables:
+      - id: items
+        type: list[text]
+      - id: item
+        type: text
+
+    inputs:
+      - items
+
+    outputs:
+      - item
+
+    steps:
+      - type: Explode
+        id: fan_out
+        inputs: [items]
+        outputs: [item]

qtype/examples/data_processing/invoke_other_flows.qtype.yaml

@@ -0,0 +1,98 @@
+id: text_analysis_pipeline
+description: Demonstrates invoking reusable flows for text analysis
+
+models:
+  - type: Model
+    id: nova_lite
+    provider: aws-bedrock
+    model_id: amazon.nova-lite-v1:0
+
+flows:
+  # Reusable flow for text summarization
+  - type: Flow
+    id: summarize_text
+    description: Summarizes input text
+    variables:
+      - id: input_text
+        type: text
+      - id: output_summary
+        type: text
+    inputs:
+      - input_text
+    outputs:
+      - output_summary
+    steps:
+      - type: LLMInference
+        id: summarize_step
+        model: nova_lite
+        system_message: "You provide concise summaries."
+        inputs: [input_text]
+        outputs: [output_summary]
+
+  # Reusable flow for sentiment analysis
+  - type: Flow
+    id: analyze_sentiment
+    description: Analyzes sentiment of text
+    variables:
+      - id: input_for_sentiment
+        type: text
+      - id: output_sentiment
+        type: text
+    inputs:
+      - input_for_sentiment
+    outputs:
+      - output_sentiment
+    steps:
+      - type: LLMInference
+        id: sentiment_step
+        model: nova_lite
+        system_message: "Analyze sentiment. Respond with only: positive, negative, or neutral."
+        inputs: [input_for_sentiment]
+        outputs: [output_sentiment]
+
+  # Main processing flow
+  - type: Flow
+    id: main
+    description: Orchestrates text analysis using multiple flows
+    variables:
+      - id: article_text
+        type: text
+      - id: summary
+        type: text
+      - id: sentiment
+        type: text
+      - id: report
+        type: text
+    inputs:
+      - article_text
+    outputs:
+      - report
+    steps:
+      # Invoke summarization flow
+      - type: InvokeFlow
+        id: get_summary
+        flow: summarize_text
+        input_bindings:
+          input_text: article_text
+        output_bindings:
+          output_summary: summary
+
+      # Invoke sentiment analysis flow
+      - type: InvokeFlow
+        id: get_sentiment
+        flow: analyze_sentiment
+        input_bindings:
+          input_for_sentiment: article_text
+        output_bindings:
+          output_sentiment: sentiment
+
+      # Combine results into a report
+      - type: PromptTemplate
+        id: create_report
+        template: |
+          Analysis Report
+          ---------------
+          Sentiment: {sentiment}
+          Summary: {summary}
+        inputs: [summary, sentiment]
+        outputs: [report]

qtype/examples/data_processing/read_file.qtype.yaml

@@ -0,0 +1,60 @@
+id: read_file_example
+description: Read data from a CSV file
+
+models:
+  - type: Model
+    id: nova
+    provider: aws-bedrock
+    model_id: amazon.nova-lite-v1:0
+
+flows:
+  - type: Flow
+    id: process_file_data
+    description: Read and process data from a CSV file
+
+    variables:
+      - id: query
+        type: text
+      - id: topic
+        type: text
+      - id: prompt
+        type: text
+      - id: answer
+        type: text
+
+    inputs: []
+
+    outputs:
+      - query
+      - topic
+      - answer
+
+    steps:
+      - id: read_data
+        type: FileSource
+        path:
+          uri: examples/data_processing/batch_inputs.csv
+        outputs:
+          - query
+          - topic
+  
+      - id: create_prompt
+        type: PromptTemplate
+        template: |
+          Topic: {topic}
+          Question: {query}
+          
+          Provide a concise answer:
+        inputs:
+          - query
+          - topic
+        outputs:
+          - prompt
+
+      - id: generate_answer
+        type: LLMInference
+        model: nova
+        inputs:
+          - prompt
+        outputs:
+          - answer

qtype/examples/data_processing/sample_article.txt

@@ -0,0 +1 @@
+Artificial intelligence is revolutionizing healthcare by enabling faster diagnoses and personalized treatment plans. Machine learning algorithms can analyze medical images with incredible accuracy, helping doctors detect diseases earlier. This technology is improving patient outcomes and reducing healthcare costs.

qtype/examples/data_processing/sample_documents.jsonl

@@ -0,0 +1,5 @@
+{"document": "The new quantum computing breakthrough could revolutionize AI processing speeds."}
+{"document": "Federal Reserve announces interest rate changes to combat inflation."}
+{"document": "New clinical trial shows promising results for cancer treatment."}
+{"document": "University expands online learning programs for remote students."}
+{"document": "Cryptocurrency markets experience volatility amid regulatory concerns."}