qtype 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qtype/` +0 -0
- qtype/application/__init__.py +0 -2
- qtype/application/converters/tools_from_api.py +28 -22
- qtype/application/converters/tools_from_module.py +66 -32
- qtype/base/__init__.py +8 -2
- qtype/base/logging.py +0 -17
- qtype/base/resources.py +193 -0
- qtype/cli.py +5 -9
- qtype/commands/generate.py +95 -7
- qtype/commands/run.py +153 -54
- qtype/docs/.pages +8 -0
- {docs → qtype/docs}/Concepts/mental-model-and-philosophy.md +1 -1
- qtype/docs/Contributing/.pages +4 -0
- {docs → qtype/docs}/Contributing/index.md +8 -1
- {docs → qtype/docs}/Gallery/dataflow_pipelines.md +18 -4
- qtype/docs/Gallery/recipe_chatbot.md +103 -0
- qtype/docs/Gallery/recipe_chatbot.mermaid +62 -0
- qtype/docs/Gallery/recipe_chatbot.png +0 -0
- {docs → qtype/docs}/Gallery/research_assistant.md +4 -5
- {docs → qtype/docs}/Gallery/simple_chatbot.md +3 -1
- {docs → qtype/docs}/How To/Authentication/configure_aws_authentication.md +2 -2
- {docs → qtype/docs}/How To/Authentication/use_api_key_authentication.md +2 -2
- {docs → qtype/docs}/How To/Command Line Usage/load_multiple_inputs_from_files.md +24 -9
- {docs → qtype/docs}/How To/Command Line Usage/pass_inputs_on_the_cli.md +7 -4
- {docs → qtype/docs}/How To/Command Line Usage/serve_with_auto_reload.md +3 -2
- {docs → qtype/docs}/How To/Data Processing/adjust_concurrency.md +3 -4
- {docs → qtype/docs}/How To/Data Processing/cache_step_results.md +2 -2
- {docs → qtype/docs}/How To/Data Processing/decode_json_xml.md +1 -1
- {docs → qtype/docs}/How To/Data Processing/explode_collections.md +2 -2
- {docs → qtype/docs}/How To/Data Processing/gather_results.md +4 -4
- qtype/docs/How To/Data Processing/invoke_other_flows.md +71 -0
- qtype/docs/How To/Data Processing/load_data_from_athena.md +49 -0
- qtype/docs/How To/Data Processing/load_documents.md +74 -0
- qtype/docs/How To/Data Processing/read_data_from_files.md +61 -0
- {docs → qtype/docs}/How To/Data Processing/read_sql_databases.md +4 -3
- {docs → qtype/docs}/How To/Data Processing/write_data_to_file.md +1 -2
- {docs → qtype/docs}/How To/Invoke Models/call_large_language_models.md +1 -1
- {docs → qtype/docs}/How To/Invoke Models/create_embeddings.md +1 -1
- {docs → qtype/docs}/How To/Invoke Models/reuse_prompts_with_templates.md +2 -3
- {docs → qtype/docs}/How To/Language Features/include_raw_text_from_other_files.md +2 -1
- {docs → qtype/docs}/How To/Language Features/reference_entities_by_id.md +2 -2
- qtype/docs/How To/Language Features/use_agent_skills.md +29 -0
- {docs → qtype/docs}/How To/Language Features/use_environment_variables.md +2 -1
- qtype/docs/How To/Language Features/use_optional_variables.md +42 -0
- {docs → qtype/docs}/How To/Language Features/use_qtype_mcp.md +4 -4
- {docs → qtype/docs}/How To/Observability & Debugging/trace_calls_with_open_telemetry.md +1 -1
- {docs → qtype/docs}/How To/Observability & Debugging/validate_qtype_yaml.md +3 -2
- {docs → qtype/docs}/How To/Observability & Debugging/visualize_application_architecture.md +1 -1
- {docs → qtype/docs}/How To/Qtype Server/serve_flows_as_apis.md +3 -3
- {docs → qtype/docs}/How To/Qtype Server/serve_flows_as_ui.md +2 -3
- {docs → qtype/docs}/How To/Qtype Server/use_conversational_interfaces.md +1 -4
- {docs → qtype/docs}/How To/Qtype Server/use_variables_with_ui_hints.md +3 -2
- {docs → qtype/docs}/How To/Tools & Integration/bind_tool_inputs_and_outputs.md +1 -2
- {docs → qtype/docs}/How To/Tools & Integration/create_tools_from_openapi_specifications.md +10 -14
- {docs → qtype/docs}/How To/Tools & Integration/create_tools_from_python_modules.md +5 -8
- {docs → qtype/docs}/Reference/cli.md +16 -17
- qtype/docs/Tutorials/.pages +1 -0
- {docs → qtype/docs}/Tutorials/01-first-qtype-application.md +4 -3
- {docs → qtype/docs}/Tutorials/02-conversational-chatbot.md +3 -3
- {docs → qtype/docs}/Tutorials/03-structured-data.md +10 -11
- {docs → qtype/docs}/Tutorials/04-tools-and-function-calling.md +13 -20
- {docs → qtype/docs}/components/APITool.md +1 -1
- qtype/docs/components/Aggregate.md +7 -0
- qtype/docs/components/Collect.md +6 -0
- qtype/docs/components/Construct.md +6 -0
- {docs → qtype/docs}/components/DocumentEmbedder.md +0 -1
- {docs → qtype/docs}/components/DocumentSplitter.md +0 -1
- qtype/docs/components/Explode.md +5 -0
- {docs → qtype/docs}/components/FieldExtractor.md +2 -1
- qtype/docs/components/InvokeFlow.md +8 -0
- qtype/docs/components/InvokeTool.md +8 -0
- {docs → qtype/docs}/components/PrimitiveTypeEnum.md +0 -1
- {docs → qtype/docs}/components/Source.md +0 -1
- {docs → qtype/docs}/components/Step.md +0 -1
- {docs → qtype/docs}/components/Tool.md +2 -2
- {docs → qtype/docs}/components/Variable.md +2 -0
- qtype/docs/legacy_how_tos/.pages +6 -0
- qtype/docs/skills/architect/SKILL.md +188 -0
- qtype/docs/skills/architect/references/cheatsheet.md +198 -0
- qtype/docs/skills/architect/references/patterns.md +29 -0
- qtype/docs/stylesheets/extra.css +27 -0
- qtype/dsl/linker.py +8 -0
- qtype/dsl/model.py +177 -84
- qtype/examples/conversational_ai/simple_chatbot_with_auth.qtype.yaml +48 -0
- qtype/examples/data_processing/athena_query.qtype.yaml +56 -0
- qtype/examples/data_processing/batch_inputs.csv +5 -0
- qtype/examples/data_processing/create_sample_db.py +129 -0
- qtype/examples/data_processing/invoke_other_flows.qtype.yaml +98 -0
- qtype/examples/data_processing/load_documents.qtype.yaml +31 -0
- qtype/examples/data_processing/reviews.db +0 -0
- qtype/examples/data_processing/sample_article.txt +1 -0
- qtype/examples/data_processing/sample_documents.jsonl +5 -0
- qtype/examples/invoke_models/invoke_embedding_aws.qtype.yaml +45 -0
- qtype/examples/language_features/optional_variables.qtype.yaml +32 -0
- qtype/examples/language_features/story_prompt.txt +6 -0
- qtype/examples/legacy/data/customers.csv +6 -0
- qtype/examples/legacy/echo/readme.md +29 -0
- qtype/examples/legacy/qtype_plugin_example.py +51 -0
- qtype/examples/legacy/sample_data.txt +43 -0
- qtype/examples/legacy/vertex/README.md +11 -0
- qtype/examples/rag/recipe_chatbot.qtype.yaml +216 -0
- qtype/examples/research_assistant/tavily.qtype.yaml +216 -0
- {examples → qtype/examples}/tutorials/03_structured_data.qtype.yaml +2 -2
- {examples → qtype/examples}/tutorials/04_tools_and_function_calling.qtype.yaml +5 -5
- qtype/interpreter/auth/aws.py +94 -17
- qtype/interpreter/auth/generic.py +11 -12
- qtype/interpreter/base/secrets.py +4 -2
- qtype/interpreter/base/stream_emitter.py +19 -13
- qtype/interpreter/conversions.py +15 -14
- qtype/interpreter/converters.py +142 -26
- qtype/interpreter/executors/agent_executor.py +2 -3
- qtype/interpreter/executors/aggregate_executor.py +3 -4
- qtype/interpreter/executors/bedrock_reranker_executor.py +17 -28
- qtype/interpreter/executors/construct_executor.py +15 -15
- qtype/interpreter/executors/doc_to_text_executor.py +1 -3
- qtype/interpreter/executors/document_embedder_executor.py +1 -12
- qtype/interpreter/executors/field_extractor_executor.py +13 -12
- qtype/interpreter/executors/file_source_executor.py +18 -31
- qtype/interpreter/executors/invoke_embedding_executor.py +24 -37
- qtype/interpreter/executors/invoke_flow_executor.py +2 -2
- qtype/interpreter/executors/invoke_tool_executor.py +19 -18
- qtype/interpreter/executors/llm_inference_executor.py +18 -18
- qtype/interpreter/executors/prompt_template_executor.py +1 -3
- qtype/interpreter/executors/sql_source_executor.py +6 -2
- qtype/interpreter/flow.py +11 -1
- qtype/interpreter/tools/function_tool_helper.py +11 -10
- qtype/interpreter/types.py +89 -4
- qtype/interpreter/typing.py +31 -32
- qtype/mcp/server.py +194 -86
- {schema → qtype/schema}/qtype.schema.json +77 -79
- qtype/semantic/checker.py +19 -0
- qtype/semantic/generate.py +3 -6
- qtype/semantic/model.py +26 -33
- qtype/semantic/resolver.py +7 -0
- qtype/semantic/visualize.py +18 -6
- {qtype-0.1.12.dist-info → qtype-0.1.14.dist-info}/METADATA +47 -46
- qtype-0.1.14.dist-info/RECORD +361 -0
- {qtype-0.1.12.dist-info → qtype-0.1.14.dist-info}/WHEEL +1 -2
- docs/How To/Data Processing/read_data_from_files.md +0 -35
- docs/components/Aggregate.md +0 -8
- docs/components/InvokeFlow.md +0 -8
- docs/components/InvokeTool.md +0 -8
- docs/components/ToolParameter.md +0 -6
- examples/research_assistant/tavily.qtype.yaml +0 -289
- qtype/application/facade.py +0 -177
- qtype-0.1.12.dist-info/RECORD +0 -325
- qtype-0.1.12.dist-info/top_level.txt +0 -1
- {docs → qtype/docs}/Contributing/roadmap.md +0 -0
- {docs → qtype/docs}/Decisions/ADR-001-Chat-vs-Completion-Endpoint-Features.md +0 -0
- {docs → qtype/docs}/Gallery/dataflow_pipelines.mermaid +0 -0
- {docs → qtype/docs}/Gallery/research_assistant.mermaid +0 -0
- {docs → qtype/docs}/Gallery/simple_chatbot.mermaid +0 -0
- {docs → qtype/docs}/How To/Language Features/include_qtype_yaml.md +0 -0
- {docs → qtype/docs}/How To/Observability & Debugging/visualize_example.mermaid +0 -0
- {docs → qtype/docs}/How To/Qtype Server/flow_as_ui.png +0 -0
- {docs → qtype/docs}/Reference/plugins.md +0 -0
- {docs → qtype/docs}/Reference/semantic-validation-rules.md +0 -0
- {docs → qtype/docs}/Tutorials/example_chat.png +0 -0
- {docs → qtype/docs}/Tutorials/index.md +0 -0
- {docs → qtype/docs}/components/APIKeyAuthProvider.md +0 -0
- {docs → qtype/docs}/components/AWSAuthProvider.md +0 -0
- {docs → qtype/docs}/components/AWSSecretManager.md +0 -0
- {docs → qtype/docs}/components/Agent.md +0 -0
- {docs → qtype/docs}/components/AggregateStats.md +0 -0
- {docs → qtype/docs}/components/Application.md +0 -0
- {docs → qtype/docs}/components/AuthorizationProvider.md +0 -0
- {docs → qtype/docs}/components/AuthorizationProviderList.md +0 -0
- {docs → qtype/docs}/components/BearerTokenAuthProvider.md +0 -0
- {docs → qtype/docs}/components/BedrockReranker.md +0 -0
- {docs → qtype/docs}/components/ChatContent.md +0 -0
- {docs → qtype/docs}/components/ChatMessage.md +0 -0
- {docs → qtype/docs}/components/ConstantPath.md +0 -0
- {docs → qtype/docs}/components/CustomType.md +0 -0
- {docs → qtype/docs}/components/Decoder.md +0 -0
- {docs → qtype/docs}/components/DecoderFormat.md +0 -0
- {docs → qtype/docs}/components/DocToTextConverter.md +0 -0
- {docs → qtype/docs}/components/Document.md +0 -0
- {docs → qtype/docs}/components/DocumentIndex.md +0 -0
- {docs → qtype/docs}/components/DocumentSearch.md +0 -0
- {docs → qtype/docs}/components/DocumentSource.md +0 -0
- {docs → qtype/docs}/components/Echo.md +0 -0
- {docs → qtype/docs}/components/Embedding.md +0 -0
- {docs → qtype/docs}/components/EmbeddingModel.md +0 -0
- {docs → qtype/docs}/components/FileSource.md +0 -0
- {docs → qtype/docs}/components/FileWriter.md +0 -0
- {docs → qtype/docs}/components/Flow.md +0 -0
- {docs → qtype/docs}/components/FlowInterface.md +0 -0
- {docs → qtype/docs}/components/Index.md +0 -0
- {docs → qtype/docs}/components/IndexUpsert.md +0 -0
- {docs → qtype/docs}/components/InvokeEmbedding.md +0 -0
- {docs → qtype/docs}/components/LLMInference.md +0 -0
- {docs → qtype/docs}/components/ListType.md +0 -0
- {docs → qtype/docs}/components/Memory.md +0 -0
- {docs → qtype/docs}/components/MessageRole.md +0 -0
- {docs → qtype/docs}/components/Model.md +0 -0
- {docs → qtype/docs}/components/ModelList.md +0 -0
- {docs → qtype/docs}/components/OAuth2AuthProvider.md +0 -0
- {docs → qtype/docs}/components/PromptTemplate.md +0 -0
- {docs → qtype/docs}/components/PythonFunctionTool.md +0 -0
- {docs → qtype/docs}/components/RAGChunk.md +0 -0
- {docs → qtype/docs}/components/RAGDocument.md +0 -0
- {docs → qtype/docs}/components/RAGSearchResult.md +0 -0
- {docs → qtype/docs}/components/Reranker.md +0 -0
- {docs → qtype/docs}/components/SQLSource.md +0 -0
- {docs → qtype/docs}/components/Search.md +0 -0
- {docs → qtype/docs}/components/SearchResult.md +0 -0
- {docs → qtype/docs}/components/SecretManager.md +0 -0
- {docs → qtype/docs}/components/SecretReference.md +0 -0
- {docs → qtype/docs}/components/TelemetrySink.md +0 -0
- {docs → qtype/docs}/components/ToolList.md +0 -0
- {docs → qtype/docs}/components/TypeList.md +0 -0
- {docs → qtype/docs}/components/VariableList.md +0 -0
- {docs → qtype/docs}/components/VectorIndex.md +0 -0
- {docs → qtype/docs}/components/VectorSearch.md +0 -0
- {docs → qtype/docs}/components/VertexAuthProvider.md +0 -0
- {docs → qtype/docs}/components/Writer.md +0 -0
- {docs → qtype/docs}/example_ui.png +0 -0
- {docs → qtype/docs}/index.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Configuration/modular-yaml.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Configuration/phoenix_projects.png +0 -0
- {docs → qtype/docs}/legacy_how_tos/Configuration/phoenix_traces.png +0 -0
- {docs → qtype/docs}/legacy_how_tos/Configuration/reference-by-id.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Configuration/telemetry-setup.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Data Types/custom-types.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Data Types/domain-types.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Debugging/visualize-apps.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Tools/api-tools.md +0 -0
- {docs → qtype/docs}/legacy_how_tos/Tools/python-tools.md +0 -0
- {examples → qtype/examples}/authentication/aws_authentication.qtype.yaml +0 -0
- {examples → qtype/examples}/conversational_ai/hello_world_chat.qtype.yaml +0 -0
- {examples → qtype/examples}/conversational_ai/simple_chatbot.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/batch_processing.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/cache_step_results.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/collect_results.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/dataflow_pipelines.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/decode_json.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/explode_items.qtype.yaml +0 -0
- {examples → qtype/examples}/data_processing/read_file.qtype.yaml +0 -0
- {examples → qtype/examples}/invoke_models/create_embeddings.qtype.yaml +0 -0
- {examples → qtype/examples}/invoke_models/simple_llm_call.qtype.yaml +0 -0
- {examples → qtype/examples}/language_features/include_raw.qtype.yaml +0 -0
- {examples → qtype/examples}/language_features/ui_hints.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/data_analysis_with_telemetry.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world_chat.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world_chat_with_telemetry.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world_chat_with_thinking.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world_completion.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/hello_world_completion_with_auth.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/bedrock/simple_agent_chat.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/chat_with_langfuse.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/data_processor.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/echo/debug_example.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/echo/prompt.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/echo/test.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/echo/video.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/field_extractor_example.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/multi_flow_example.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/openai/hello_world_chat.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/openai/hello_world_chat_with_telemetry.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/rag.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/time_utilities.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/vertex/hello_world_chat.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/vertex/hello_world_completion.qtype.yaml +0 -0
- {examples → qtype/examples}/legacy/vertex/hello_world_completion_with_auth.qtype.yaml +0 -0
- {examples → qtype/examples}/observability_debugging/trace_with_opentelemetry.qtype.yaml +0 -0
- {examples → qtype/examples}/research_assistant/research_assistant.qtype.yaml +0 -0
- {examples → qtype/examples}/research_assistant/tavily.oas.yaml +0 -0
- {examples → qtype/examples}/tutorials/01_hello_world.qtype.yaml +0 -0
- {examples → qtype/examples}/tutorials/02_conversational_chat.qtype.yaml +0 -0
- {qtype-0.1.12.dist-info → qtype-0.1.14.dist-info}/entry_points.txt +0 -0
- {qtype-0.1.12.dist-info → qtype-0.1.14.dist-info}/licenses/LICENSE +0 -0
qtype/commands/generate.py
CHANGED
|
@@ -2,24 +2,48 @@ import argparse
|
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Optional
|
|
5
|
+
from typing import Any, Optional
|
|
6
6
|
|
|
7
7
|
from qtype.dsl.model import Document
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
11
|
|
|
12
|
+
def generate_aws_bedrock_models() -> list[dict[str, Any]]:
|
|
13
|
+
"""Generate AWS Bedrock model definitions.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
List of model definitions for AWS Bedrock models.
|
|
17
|
+
|
|
18
|
+
Raises:
|
|
19
|
+
ImportError: If boto3 is not installed.
|
|
20
|
+
Exception: If AWS API call fails.
|
|
21
|
+
"""
|
|
22
|
+
import boto3 # type: ignore[import-untyped]
|
|
23
|
+
|
|
24
|
+
logger.info("Discovering AWS Bedrock models...")
|
|
25
|
+
client = boto3.client("bedrock")
|
|
26
|
+
models = client.list_foundation_models()
|
|
27
|
+
|
|
28
|
+
model_definitions = []
|
|
29
|
+
for model_summary in models.get("modelSummaries", []):
|
|
30
|
+
model_definitions.append(
|
|
31
|
+
{
|
|
32
|
+
"id": model_summary["modelId"],
|
|
33
|
+
"provider": "aws-bedrock",
|
|
34
|
+
}
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
logger.info(f"Discovered {len(model_definitions)} AWS Bedrock models")
|
|
38
|
+
return model_definitions
|
|
39
|
+
|
|
40
|
+
|
|
12
41
|
def run_dump_commons_library(args: argparse.Namespace) -> None:
|
|
13
42
|
"""Generate commons library tools and AWS Bedrock models."""
|
|
14
|
-
import logging
|
|
15
43
|
from pathlib import Path
|
|
16
44
|
|
|
17
|
-
from qtype.application.facade import QTypeFacade
|
|
18
45
|
from qtype.dsl.model import Model, ModelList
|
|
19
46
|
|
|
20
|
-
logger = logging.getLogger(__name__)
|
|
21
|
-
facade = QTypeFacade()
|
|
22
|
-
|
|
23
47
|
try:
|
|
24
48
|
# Generate common tools using convert module functionality
|
|
25
49
|
logger.info("Generating common tools...")
|
|
@@ -38,7 +62,7 @@ def run_dump_commons_library(args: argparse.Namespace) -> None:
|
|
|
38
62
|
# Generate AWS Bedrock models
|
|
39
63
|
logger.info("Generating AWS Bedrock models...")
|
|
40
64
|
try:
|
|
41
|
-
model_definitions =
|
|
65
|
+
model_definitions = generate_aws_bedrock_models()
|
|
42
66
|
|
|
43
67
|
model_list = ModelList(
|
|
44
68
|
root=[
|
|
@@ -80,6 +104,54 @@ def run_generate_documentation(args: argparse.Namespace) -> None:
|
|
|
80
104
|
generate_documentation(Path(args.output))
|
|
81
105
|
|
|
82
106
|
|
|
107
|
+
def _copy_resource_file(resource, rel_path: Path, output_file: Path) -> None:
|
|
108
|
+
"""Copy a file from a resource directory to an output location."""
|
|
109
|
+
content = resource.get_file(str(rel_path))
|
|
110
|
+
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
output_file.write_text(content, encoding="utf-8")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def run_generate_skill(args: argparse.Namespace) -> None:
|
|
115
|
+
"""Generate a Claude skill with QType documentation and examples.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
args: Command-line arguments with 'output' path.
|
|
119
|
+
"""
|
|
120
|
+
from qtype.base.resources import get_docs_resource, get_examples_resource
|
|
121
|
+
|
|
122
|
+
_docs_resource = get_docs_resource()
|
|
123
|
+
_examples_resource = get_examples_resource()
|
|
124
|
+
|
|
125
|
+
output_path = Path(args.output) / "qtype-architect"
|
|
126
|
+
|
|
127
|
+
# Copy skill documentation files
|
|
128
|
+
skills_path = _docs_resource.get_path() / "skills" / "architect"
|
|
129
|
+
skill_count = 0
|
|
130
|
+
for skill_file in skills_path.rglob("*.*"):
|
|
131
|
+
rel_path = skill_file.relative_to(_docs_resource.get_path())
|
|
132
|
+
_copy_resource_file(
|
|
133
|
+
_docs_resource,
|
|
134
|
+
rel_path,
|
|
135
|
+
output_path / rel_path.relative_to("skills/architect"),
|
|
136
|
+
)
|
|
137
|
+
skill_count += 1
|
|
138
|
+
|
|
139
|
+
# Copy all example files
|
|
140
|
+
example_count = 0
|
|
141
|
+
for yaml_file in _examples_resource.get_path().rglob("*.yaml"):
|
|
142
|
+
rel_path = yaml_file.relative_to(_examples_resource.get_path())
|
|
143
|
+
if "legacy" not in rel_path.parts:
|
|
144
|
+
_copy_resource_file(
|
|
145
|
+
_examples_resource, rel_path, output_path / "assets" / rel_path
|
|
146
|
+
)
|
|
147
|
+
example_count += 1
|
|
148
|
+
|
|
149
|
+
logger.info(
|
|
150
|
+
f"Skill generated at {output_path}: "
|
|
151
|
+
f"{skill_count} docs, {example_count} examples"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
83
155
|
def generate_schema(args: argparse.Namespace) -> None:
|
|
84
156
|
"""Generate and output the JSON schema for Document.
|
|
85
157
|
|
|
@@ -87,6 +159,7 @@ def generate_schema(args: argparse.Namespace) -> None:
|
|
|
87
159
|
args (argparse.Namespace): Command-line arguments with an optional
|
|
88
160
|
'output' attribute specifying the output file path.
|
|
89
161
|
"""
|
|
162
|
+
logger.info("Generating QType DSL JSON schema...")
|
|
90
163
|
schema = Document.model_json_schema()
|
|
91
164
|
|
|
92
165
|
# Add the $schema property to indicate JSON Schema version
|
|
@@ -166,6 +239,7 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
166
239
|
"-o",
|
|
167
240
|
"--output",
|
|
168
241
|
type=str,
|
|
242
|
+
default=None,
|
|
169
243
|
help="Output file for the schema (default: stdout)",
|
|
170
244
|
)
|
|
171
245
|
schema_parser.set_defaults(func=generate_schema)
|
|
@@ -184,6 +258,20 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
184
258
|
)
|
|
185
259
|
dsl_parser.set_defaults(func=run_generate_documentation)
|
|
186
260
|
|
|
261
|
+
# Parser for generating Agent skills
|
|
262
|
+
skill_parser = generate_subparsers.add_parser(
|
|
263
|
+
"skills",
|
|
264
|
+
help="Generates Agent skills with QType documentation and examples.",
|
|
265
|
+
)
|
|
266
|
+
skill_parser.add_argument(
|
|
267
|
+
"-o",
|
|
268
|
+
"--output",
|
|
269
|
+
type=str,
|
|
270
|
+
default=".claude/skills",
|
|
271
|
+
help="Output directory for the skills (default: .claude/skills)",
|
|
272
|
+
)
|
|
273
|
+
skill_parser.set_defaults(func=run_generate_skill)
|
|
274
|
+
|
|
187
275
|
# Parser for generating the semantic model
|
|
188
276
|
# only add this if networkx and ruff are installed
|
|
189
277
|
try:
|
qtype/commands/run.py
CHANGED
|
@@ -11,11 +11,10 @@ import warnings
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
-
import pandas as pd
|
|
15
14
|
from pydantic.warnings import UnsupportedFieldAttributeWarning
|
|
16
15
|
|
|
17
|
-
from qtype.application.facade import QTypeFacade
|
|
18
16
|
from qtype.base.exceptions import InterpreterError, LoadError, ValidationError
|
|
17
|
+
from qtype.interpreter.converters import read_dataframe_from_file
|
|
19
18
|
|
|
20
19
|
logger = logging.getLogger(__name__)
|
|
21
20
|
|
|
@@ -29,50 +28,124 @@ for name in ["httpx", "urllib3", "qdrant_client", "opensearch"]:
|
|
|
29
28
|
logging.getLogger(name).setLevel(logging.WARNING)
|
|
30
29
|
|
|
31
30
|
|
|
32
|
-
def
|
|
33
|
-
"""
|
|
34
|
-
|
|
31
|
+
def register_telemetry(spec) -> None:
|
|
32
|
+
"""Register telemetry if enabled in the spec."""
|
|
33
|
+
from qtype.interpreter.telemetry import register
|
|
34
|
+
from qtype.semantic.model import Application as SemanticApplication
|
|
35
|
+
|
|
36
|
+
if isinstance(spec, SemanticApplication) and spec.telemetry:
|
|
37
|
+
logger.info(
|
|
38
|
+
f"Telemetry enabled with endpoint: {spec.telemetry.endpoint}"
|
|
39
|
+
)
|
|
40
|
+
secret_mgr = create_secret_manager_for_spec(spec)
|
|
41
|
+
register(spec.telemetry, secret_mgr, spec.id)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def create_secret_manager_for_spec(spec):
|
|
45
|
+
"""Create a secret manager based on the specification."""
|
|
46
|
+
from qtype.interpreter.base.secrets import create_secret_manager
|
|
47
|
+
from qtype.semantic.model import Application as SemanticApplication
|
|
48
|
+
|
|
49
|
+
if isinstance(spec, SemanticApplication):
|
|
50
|
+
return create_secret_manager(spec.secret_manager)
|
|
51
|
+
else:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
"Can't create secret manager for non-Application spec"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def execute_workflow(
|
|
58
|
+
path: Path,
|
|
59
|
+
inputs: dict | Any,
|
|
60
|
+
flow_name: str | None = None,
|
|
61
|
+
**kwargs: Any,
|
|
62
|
+
) -> Any:
|
|
63
|
+
"""Execute a complete workflow from document to results.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
path: Path to the QType specification file
|
|
67
|
+
inputs: Dictionary of input values or DataFrame for batch
|
|
68
|
+
flow_name: Optional name of flow to execute
|
|
69
|
+
**kwargs: Additional dependencies for execution
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
DataFrame with results (one row per input)
|
|
35
73
|
"""
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
74
|
+
import pandas as pd
|
|
75
|
+
from opentelemetry import trace
|
|
76
|
+
|
|
77
|
+
from qtype.interpreter.base.executor_context import ExecutorContext
|
|
78
|
+
from qtype.interpreter.converters import (
|
|
79
|
+
dataframe_to_flow_messages,
|
|
80
|
+
flow_messages_to_dataframe,
|
|
81
|
+
)
|
|
82
|
+
from qtype.interpreter.flow import run_flow
|
|
83
|
+
from qtype.interpreter.types import Session
|
|
84
|
+
from qtype.semantic.loader import load
|
|
85
|
+
from qtype.semantic.model import Application as SemanticApplication
|
|
86
|
+
|
|
87
|
+
# Load the semantic application
|
|
88
|
+
semantic_model, type_registry = load(path)
|
|
89
|
+
assert isinstance(semantic_model, SemanticApplication)
|
|
90
|
+
register_telemetry(semantic_model)
|
|
91
|
+
|
|
92
|
+
# Find the flow to execute
|
|
93
|
+
if flow_name:
|
|
94
|
+
target_flow = None
|
|
95
|
+
for flow in semantic_model.flows:
|
|
96
|
+
if flow.id == flow_name:
|
|
97
|
+
target_flow = flow
|
|
98
|
+
break
|
|
99
|
+
if target_flow is None:
|
|
100
|
+
raise ValueError(f"Flow '{flow_name}' not found")
|
|
101
|
+
else:
|
|
102
|
+
if semantic_model.flows:
|
|
103
|
+
target_flow = semantic_model.flows[0]
|
|
55
104
|
else:
|
|
56
|
-
raise ValueError(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
66
|
-
"application/vnd.ms-excel",
|
|
67
|
-
]:
|
|
68
|
-
return pd.read_excel(file_path)
|
|
69
|
-
elif mime_type in ["application/vnd.parquet", "application/octet-stream"]:
|
|
70
|
-
return pd.read_parquet(file_path)
|
|
105
|
+
raise ValueError("No flows found in application")
|
|
106
|
+
|
|
107
|
+
logger.info(f"Executing flow {target_flow.id} from {path}")
|
|
108
|
+
|
|
109
|
+
# Convert inputs to DataFrame (normalize single dict to 1-row DataFrame)
|
|
110
|
+
if isinstance(inputs, dict):
|
|
111
|
+
input_df = pd.DataFrame([inputs])
|
|
112
|
+
elif isinstance(inputs, pd.DataFrame):
|
|
113
|
+
input_df = inputs
|
|
71
114
|
else:
|
|
72
115
|
raise ValueError(
|
|
73
|
-
f"
|
|
116
|
+
f"Inputs must be dict or DataFrame, got {type(inputs)}"
|
|
74
117
|
)
|
|
75
118
|
|
|
119
|
+
# Create session
|
|
120
|
+
session = Session(
|
|
121
|
+
session_id=kwargs.pop("session_id", "default"),
|
|
122
|
+
conversation_history=kwargs.pop("conversation_history", []),
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# Convert DataFrame to FlowMessages with type conversion
|
|
126
|
+
initial_messages_list = dataframe_to_flow_messages(
|
|
127
|
+
input_df, target_flow.inputs, session=session
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# Execute the flow
|
|
131
|
+
secret_manager = create_secret_manager_for_spec(semantic_model)
|
|
132
|
+
|
|
133
|
+
context = ExecutorContext(
|
|
134
|
+
secret_manager=secret_manager,
|
|
135
|
+
tracer=trace.get_tracer(__name__),
|
|
136
|
+
)
|
|
137
|
+
results = await run_flow(
|
|
138
|
+
target_flow,
|
|
139
|
+
initial_messages_list,
|
|
140
|
+
context=context,
|
|
141
|
+
**kwargs,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# Convert results back to DataFrame
|
|
145
|
+
results_df = flow_messages_to_dataframe(results, target_flow)
|
|
146
|
+
|
|
147
|
+
return results_df
|
|
148
|
+
|
|
76
149
|
|
|
77
150
|
def run_flow(args: Any) -> None:
|
|
78
151
|
"""Run a QType YAML spec file by executing its flows.
|
|
@@ -82,7 +155,6 @@ def run_flow(args: Any) -> None:
|
|
|
82
155
|
"""
|
|
83
156
|
import asyncio
|
|
84
157
|
|
|
85
|
-
facade = QTypeFacade()
|
|
86
158
|
spec_path = Path(args.spec)
|
|
87
159
|
|
|
88
160
|
try:
|
|
@@ -90,7 +162,7 @@ def run_flow(args: Any) -> None:
|
|
|
90
162
|
|
|
91
163
|
if args.input_file:
|
|
92
164
|
logger.info(f"Loading input data from file: {args.input_file}")
|
|
93
|
-
input: Any =
|
|
165
|
+
input: Any = read_dataframe_from_file(args.input_file)
|
|
94
166
|
else:
|
|
95
167
|
# Parse input JSON
|
|
96
168
|
try:
|
|
@@ -99,9 +171,9 @@ def run_flow(args: Any) -> None:
|
|
|
99
171
|
logger.error(f"❌ Invalid JSON input: {e}")
|
|
100
172
|
return
|
|
101
173
|
|
|
102
|
-
# Execute the workflow using the
|
|
174
|
+
# Execute the workflow using the standalone function
|
|
103
175
|
result_df = asyncio.run(
|
|
104
|
-
|
|
176
|
+
execute_workflow(
|
|
105
177
|
spec_path,
|
|
106
178
|
flow_name=args.flow,
|
|
107
179
|
inputs=input,
|
|
@@ -113,7 +185,7 @@ def run_flow(args: Any) -> None:
|
|
|
113
185
|
|
|
114
186
|
# Display results
|
|
115
187
|
if len(result_df) > 0:
|
|
116
|
-
logger.info(f"Processed {len(result_df)}
|
|
188
|
+
logger.info(f"Processed {len(result_df)} rows")
|
|
117
189
|
|
|
118
190
|
# Remove 'row' and 'error' columns for display if all errors are None
|
|
119
191
|
display_df = result_df.copy()
|
|
@@ -125,15 +197,37 @@ def run_flow(args: Any) -> None:
|
|
|
125
197
|
if "row" in display_df.columns:
|
|
126
198
|
display_df = display_df.drop(columns=["row"])
|
|
127
199
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
200
|
+
# Show summary for console display
|
|
201
|
+
logger.info(
|
|
202
|
+
f"\nResults summary: {len(display_df)} rows, "
|
|
203
|
+
f"{len(display_df.columns)} columns: {list(display_df.columns)}"
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# Optionally show full output
|
|
207
|
+
if args.show_output:
|
|
208
|
+
# Truncate long strings for display
|
|
209
|
+
max_col_width = 100
|
|
210
|
+
for col in display_df.columns:
|
|
211
|
+
display_df[col] = display_df[col].apply(
|
|
212
|
+
lambda x: (
|
|
213
|
+
f"{str(x)[:max_col_width]}..."
|
|
214
|
+
if isinstance(x, str)
|
|
215
|
+
and len(str(x)) > max_col_width
|
|
216
|
+
else x
|
|
217
|
+
)
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if len(display_df) > 1:
|
|
221
|
+
logger.info(
|
|
222
|
+
f"\nResults:\n{display_df[0:10].to_string()}\n..."
|
|
223
|
+
)
|
|
224
|
+
else:
|
|
225
|
+
# Print the first row with column_name: value one per line
|
|
226
|
+
fmt_str = []
|
|
227
|
+
for col, val in display_df.iloc[0].items():
|
|
228
|
+
fmt_str.append(f"{col}: {val}")
|
|
229
|
+
fmt_str = "\n".join(fmt_str)
|
|
230
|
+
logger.info(f"\nResults:\n{fmt_str}")
|
|
137
231
|
|
|
138
232
|
# Save the output
|
|
139
233
|
if args.output:
|
|
@@ -195,6 +289,11 @@ def parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
195
289
|
action="store_true",
|
|
196
290
|
help="Show progress bars during flow execution.",
|
|
197
291
|
)
|
|
292
|
+
cmd_parser.add_argument(
|
|
293
|
+
"--show-output",
|
|
294
|
+
action="store_true",
|
|
295
|
+
help="Display full output data in console (default: summary only).",
|
|
296
|
+
)
|
|
198
297
|
|
|
199
298
|
cmd_parser.add_argument(
|
|
200
299
|
"spec", type=str, help="Path to the QType YAML spec file."
|
qtype/docs/.pages
ADDED
|
@@ -128,7 +128,7 @@ Output Variables
|
|
|
128
128
|
|
|
129
129
|
**Linear execution:** Steps run sequentially in declaration order. Each step waits for its inputs to be available. Parallelism is supported for multiple inputs.
|
|
130
130
|
|
|
131
|
-
**1-to-many cardinality:** Some steps (like `Explode`) can produce multiple outputs for one input, creating fan-out patterns. Other steps (like `Collect`)
|
|
131
|
+
**1-to-many cardinality:** Some steps (like `Explode`) can produce multiple outputs for one input, creating fan-out patterns. Other steps (like `Collect`) gather many inputs into one collection. This enables batch processing patterns.
|
|
132
132
|
|
|
133
133
|
---
|
|
134
134
|
|
|
@@ -57,6 +57,14 @@ After installation, you should be able to run the `qtype` command from anywhere:
|
|
|
57
57
|
qtype --help
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
+
## Running the MCP In Dev Mode
|
|
61
|
+
|
|
62
|
+
To start it in `dev` mode with the inspector:
|
|
63
|
+
```
|
|
64
|
+
mcp dev qtype/mcp/server.py:mcp
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
|
|
60
68
|
## Running Tests
|
|
61
69
|
|
|
62
70
|
The project uses pytest for testing with coverage measurement:
|
|
@@ -161,7 +169,6 @@ pre-commit install
|
|
|
161
169
|
|
|
162
170
|
Settings are in `.pre-commit-config.yaml`:
|
|
163
171
|
|
|
164
|
-
|
|
165
172
|
## Project Structure
|
|
166
173
|
|
|
167
174
|
- `qtype/` – Python package for parsing, validating, and interpreting QType specs
|
|
@@ -64,11 +64,24 @@ Example output:
|
|
|
64
64
|
|
|
65
65
|
You'll notice that the output shows 1 message for `write_results` and 10 for the others. That is because it is reporting the number of messages _emitted_ from each step, and `write_results` is a sink that collects all messages.
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
By default, QType shows a summary of the results. The final message will show:
|
|
68
68
|
|
|
69
69
|
```
|
|
70
70
|
2026-01-16 11:23:35,151 - INFO: ✅ Flow execution completed successfully
|
|
71
|
-
2026-01-16 11:23:35,151 - INFO: Processed 1
|
|
71
|
+
2026-01-16 11:23:35,151 - INFO: Processed 1 rows
|
|
72
|
+
2026-01-16 11:23:35,152 - INFO:
|
|
73
|
+
Results summary: 1 rows, 1 columns: ['result_file']
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
To see the full output data, add the `--show-output` flag:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
qtype run -i '{"output_path":"results.parquet"}' --progress --show-output examples/data_processing/dataflow_pipelines.qtype.yaml
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
This will display:
|
|
83
|
+
|
|
84
|
+
```
|
|
72
85
|
2026-01-16 11:23:35,152 - INFO:
|
|
73
86
|
Results:
|
|
74
87
|
result_file: results.parquet
|
|
@@ -76,5 +89,6 @@ result_file: results.parquet
|
|
|
76
89
|
|
|
77
90
|
## Learn More
|
|
78
91
|
|
|
79
|
-
- Tutorial:
|
|
80
|
-
-
|
|
92
|
+
- [Tutorial: Your First QType Application](../Tutorials/01-first-qtype-application.md)
|
|
93
|
+
- [Read Data from SQL Databases](../How%20To/Data%20Processing/read_sql_databases.md)
|
|
94
|
+
- [Adjust Concurrency](../How%20To/Data%20Processing/adjust_concurrency.md)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Retrieval Augmented Generation Chatbot
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
|
|
5
|
+
A complete RAG (Retrieval Augmented Generation) chatbot that answers cooking questions using a recipe collection from GitHub. The system ingests markdown recipe files, splits them into chunks, generates embeddings, stores them in a vector database, and provides conversational search with context-aware responses using memory to maintain conversation history.
|
|
6
|
+
|
|
7
|
+
## Architecture
|
|
8
|
+
|
|
9
|
+
```mermaid
|
|
10
|
+
--8<-- "Gallery/recipe_chatbot.mermaid"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Complete Code
|
|
14
|
+
|
|
15
|
+
```yaml
|
|
16
|
+
--8<-- "../examples/rag/recipe_chatbot.qtype.yaml"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Running the Example
|
|
20
|
+
|
|
21
|
+
### Prerequisites
|
|
22
|
+
|
|
23
|
+
Start Qdrant vector database locally:
|
|
24
|
+
```bash
|
|
25
|
+
docker run -p 6333:6333 qdrant/qdrant
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Clone the recipe repository:
|
|
29
|
+
```bash
|
|
30
|
+
git clone https://github.com/clarklab/chowdown.git
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Ingest Recipe Documents
|
|
34
|
+
|
|
35
|
+
Run the ingestion flow to populate the vector index:
|
|
36
|
+
```bash
|
|
37
|
+
AWS_PROFILE=my_profile qtype run examples/rag/recipe_chatbot.qtype.yaml --flow recipe_ingestion --progress
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
This will:
|
|
41
|
+
1. Load all markdown files from `chowdown/_recipes/`
|
|
42
|
+
2. Split them into 512-token chunks with 50-token overlap
|
|
43
|
+
3. Generate embeddings using AWS Bedrock Titan
|
|
44
|
+
4. Store vectors in Qdrant collection `chowdown_recipes`
|
|
45
|
+
|
|
46
|
+
You should see the output similar to:
|
|
47
|
+
```
|
|
48
|
+
2026-02-04 06:38:06,222 - qtype.commands.run - INFO - Running flow from recipe_chatbot.qtype.yaml
|
|
49
|
+
2026-02-04 06:38:06,315 - qtype.commands.run - INFO - Executing flow recipe_ingestion from recipe_chatbot.qtype.yaml
|
|
50
|
+
/Users/lou.kratz/repos/qtype-cicd-fix/.venv/lib/python3.13/site-packages/llama_index/vector_stores/qdrant/base.py:238: UserWarning: Api key is used with an insecure connection.
|
|
51
|
+
self._client = qdrant_client.QdrantClient(
|
|
52
|
+
/Users/lou.kratz/repos/qtype-cicd-fix/.venv/lib/python3.13/site-packages/llama_index/vector_stores/qdrant/base.py:241: UserWarning: Api key is used with an insecure connection.
|
|
53
|
+
self._aclient = qdrant_client.AsyncQdrantClient(
|
|
54
|
+
╭─────────────────────────────────────────────── Flow Progress ────────────────────────────────────────────────╮
|
|
55
|
+
│ │
|
|
56
|
+
│ Step load_recipes 12.9 msg/s ▁▁▂▄▄▅▅▅▅▄▆▆▆▇▇█▇▇… ✔ 34 succeeded ✖ 0 errors ⟳ - hits ✗ - misses 0:00:02 │
|
|
57
|
+
│ Step split_recipes 14.9 msg/s ▁▁▁▃▂▅▅▅▆▅▆▆▇▇▇█▇▇… ✔ 39 succeeded ✖ 0 errors ⟳ - hits ✗ - misses 0:00:02 │
|
|
58
|
+
│ Step embed_chunks 18.7 msg/s ██▃▃▁▂▂▁▂▁▁▁▁▁▁▁▁▁… ✔ 39 succeeded ✖ 0 errors ⟳ - hits ✗ - misses 0:00:02 │
|
|
59
|
+
│ Step index_recipes 47.0 msg/s ████████▁ ✔ 39 succeeded ✖ 0 errors ⟳ - hits ✗ - misses 0:00:00 │
|
|
60
|
+
│ │
|
|
61
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
62
|
+
2026-02-04 06:38:11,141 - qtype.commands.run - INFO - ✅ Flow execution completed successfully
|
|
63
|
+
2026-02-04 06:38:11,141 - qtype.commands.run - INFO - Processed 39 rows
|
|
64
|
+
2026-02-04 06:38:11,141 - qtype.commands.run - INFO -
|
|
65
|
+
Results summary: 39 rows, 1 columns: ['embedded_chunk']
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Start the Chatbot
|
|
69
|
+
|
|
70
|
+
Launch the conversational UI:
|
|
71
|
+
```bash
|
|
72
|
+
AWS_PROFILE=my_profile qtype serve examples/rag/recipe_chatbot.qtype.yaml --flow recipe_chat
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Then open http://localhost:8000 and ask questions like:
|
|
76
|
+
- "What dessert recipes do you have?"
|
|
77
|
+
- "What can I make with chicken?"
|
|
78
|
+
|
|
79
|
+

|
|
80
|
+
|
|
81
|
+
## Key Features
|
|
82
|
+
|
|
83
|
+
- **Conversational Interface**: Flow interface type that accumulates messages in `conversation_history` for stateful multi-turn chat
|
|
84
|
+
- **Memory**: Conversation buffer with `token_limit` (10,000) and `chat_history_token_ratio` (0.7) that auto-flushes oldest messages when limit exceeded
|
|
85
|
+
- **DocumentSource**: Loads markdown files via LlamaIndex `SimpleDirectoryReader` with `required_exts` file filter
|
|
86
|
+
- **DocumentSplitter**: Splits documents with `SentenceSplitter` using `chunk_size` (512) and `chunk_overlap` (50) parameters
|
|
87
|
+
- **DocumentEmbedder**: Generates embeddings with AWS Bedrock Titan, processes chunks concurrently via `num_workers` (5)
|
|
88
|
+
- **VectorIndex**: Qdrant vector store with `embedding_model` reference and dimensions (1024)
|
|
89
|
+
- **IndexUpsert**: Writes to vector index in batches via `batch_size` (25)
|
|
90
|
+
- **VectorSearch**: Semantic search with `default_top_k` (5) returns chunks by embedding distance
|
|
91
|
+
- **FieldExtractor**: Extracts text from ChatMessage using JSONPath `$.blocks[?(@.type == 'text')].content`
|
|
92
|
+
- **PromptTemplate**: Injects search results and query into template string for LLM context
|
|
93
|
+
- **LLMInference**: Calls model with `system_message` and `memory` reference for conversation history
|
|
94
|
+
- **RAGDocument**: Domain type with `content`, `file_id`, `file_name`, `metadata` fields
|
|
95
|
+
- **RAGChunk**: Domain type with `content`, `chunk_id`, `document_id`, `vector` fields
|
|
96
|
+
- **RAGSearchResult**: Domain type with `content` (RAGChunk), `doc_id`, `score` fields
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
## Learn More
|
|
100
|
+
|
|
101
|
+
- Tutorial: [Building a Stateful Chatbot](../../Tutorials/building-a-stateful-chatbot.md)
|
|
102
|
+
- How-To: [Use Environment Variables](../../How-To%20Guides/Language%20Features/use-environment-variables.md)
|
|
103
|
+
- How-To: [Configure AWS Authentication](../../How-To%20Guides/Authentication/configure-aws-authentication.md)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
flowchart TD
|
|
2
|
+
subgraph APP ["📱 recipe_rag_chatbot"]
|
|
3
|
+
direction TB
|
|
4
|
+
|
|
5
|
+
subgraph FLOW_0 ["🔄 recipe_chat"]
|
|
6
|
+
direction LR
|
|
7
|
+
FLOW_0_START@{shape: circle, label: "▶️ Start"}
|
|
8
|
+
FLOW_0_S0@{shape: rect, label: "⚙️ extract_question"}
|
|
9
|
+
FLOW_0_S1@{shape: cyl, label: "🔎 search_recipes"}
|
|
10
|
+
FLOW_0_S2@{shape: doc, label: "📄 build_context_prompt"}
|
|
11
|
+
FLOW_0_S3@{shape: rounded, label: "✨ generate_response"}
|
|
12
|
+
FLOW_0_START -->|user_message| FLOW_0_S0
|
|
13
|
+
FLOW_0_S0 -->|user_question| FLOW_0_S1
|
|
14
|
+
FLOW_0_S1 -->|search_results| FLOW_0_S2
|
|
15
|
+
FLOW_0_S0 -->|user_question| FLOW_0_S2
|
|
16
|
+
FLOW_0_S2 -->|context_prompt| FLOW_0_S3
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
subgraph FLOW_1 ["🔄 recipe_ingestion"]
|
|
20
|
+
direction TB
|
|
21
|
+
FLOW_1_S0@{shape: rect, label: "⚙️ load_recipes"}
|
|
22
|
+
FLOW_1_S1@{shape: rect, label: "⚙️ split_recipes"}
|
|
23
|
+
FLOW_1_S2@{shape: rect, label: "⚙️ embed_chunks"}
|
|
24
|
+
FLOW_1_S3@{shape: rect, label: "💾 index_recipes"}
|
|
25
|
+
FLOW_1_S0 -->|recipe_document| FLOW_1_S1
|
|
26
|
+
FLOW_1_S1 -->|recipe_chunk| FLOW_1_S2
|
|
27
|
+
FLOW_1_S2 -->|embedded_chunk| FLOW_1_S3
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
subgraph RESOURCES ["🔧 Shared Resources"]
|
|
31
|
+
direction LR
|
|
32
|
+
AUTH_AWS_AUTH@{shape: hex, label: "🔐 aws_auth (AWS)"}
|
|
33
|
+
MODEL_CLAUDE_SONNET@{shape: rounded, label: "✨ claude_sonnet (aws-bedrock)" }
|
|
34
|
+
MODEL_CLAUDE_SONNET -.->|uses| AUTH_AWS_AUTH
|
|
35
|
+
MODEL_TITAN_EMBED@{shape: rounded, label: "✨ titan_embed (aws-bedrock)" }
|
|
36
|
+
MODEL_TITAN_EMBED -.->|uses| AUTH_AWS_AUTH
|
|
37
|
+
INDEX_RECIPE_INDEX@{shape: cyl, label: "�️ recipe_index"}
|
|
38
|
+
EMB_TITAN_EMBED@{shape: rounded, label: "🎯 titan_embed"}
|
|
39
|
+
INDEX_RECIPE_INDEX -.->|embeds| EMB_TITAN_EMBED
|
|
40
|
+
MEM_RECIPE_CHAT_MEMORY@{shape: win-pane, label: "🧠 recipe_chat_memory (10KT)"}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
FLOW_0_S1 -.-> INDEX_RECIPE_INDEX
|
|
46
|
+
FLOW_0_S3 -.->|uses| MODEL_CLAUDE_SONNET
|
|
47
|
+
FLOW_0_S3 -.->|stores| MEM_RECIPE_CHAT_MEMORY
|
|
48
|
+
FLOW_1_S3 -.->|writes| INDEX_RECIPE_INDEX
|
|
49
|
+
|
|
50
|
+
%% Styling
|
|
51
|
+
classDef appBox fill:none,stroke:#495057,stroke-width:3px
|
|
52
|
+
classDef flowBox fill:#e1f5fe,stroke:#0277bd,stroke-width:2px
|
|
53
|
+
classDef llmNode fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
|
|
54
|
+
classDef modelNode fill:#e8f5e8,stroke:#2e7d32,stroke-width:2px
|
|
55
|
+
classDef authNode fill:#fff3e0,stroke:#ef6c00,stroke-width:2px
|
|
56
|
+
classDef telemetryNode fill:#fce4ec,stroke:#c2185b,stroke-width:2px
|
|
57
|
+
classDef resourceBox fill:#f5f5f5,stroke:#616161,stroke-width:1px
|
|
58
|
+
|
|
59
|
+
class APP appBox
|
|
60
|
+
class FLOW_0 flowBox
|
|
61
|
+
class RESOURCES resourceBox
|
|
62
|
+
class TELEMETRY telemetryNode
|