hexdag 0.5.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hexdag/__init__.py +116 -0
- hexdag/__main__.py +30 -0
- hexdag/adapters/executors/__init__.py +5 -0
- hexdag/adapters/executors/local_executor.py +316 -0
- hexdag/builtin/__init__.py +6 -0
- hexdag/builtin/adapters/__init__.py +51 -0
- hexdag/builtin/adapters/anthropic/__init__.py +5 -0
- hexdag/builtin/adapters/anthropic/anthropic_adapter.py +151 -0
- hexdag/builtin/adapters/database/__init__.py +6 -0
- hexdag/builtin/adapters/database/csv/csv_adapter.py +249 -0
- hexdag/builtin/adapters/database/pgvector/__init__.py +5 -0
- hexdag/builtin/adapters/database/pgvector/pgvector_adapter.py +478 -0
- hexdag/builtin/adapters/database/sqlalchemy/sqlalchemy_adapter.py +252 -0
- hexdag/builtin/adapters/database/sqlite/__init__.py +5 -0
- hexdag/builtin/adapters/database/sqlite/sqlite_adapter.py +410 -0
- hexdag/builtin/adapters/local/README.md +59 -0
- hexdag/builtin/adapters/local/__init__.py +7 -0
- hexdag/builtin/adapters/local/local_observer_manager.py +696 -0
- hexdag/builtin/adapters/memory/__init__.py +47 -0
- hexdag/builtin/adapters/memory/file_memory_adapter.py +297 -0
- hexdag/builtin/adapters/memory/in_memory_memory.py +216 -0
- hexdag/builtin/adapters/memory/schemas.py +57 -0
- hexdag/builtin/adapters/memory/session_memory.py +178 -0
- hexdag/builtin/adapters/memory/sqlite_memory_adapter.py +215 -0
- hexdag/builtin/adapters/memory/state_memory.py +280 -0
- hexdag/builtin/adapters/mock/README.md +89 -0
- hexdag/builtin/adapters/mock/__init__.py +15 -0
- hexdag/builtin/adapters/mock/hexdag.toml +50 -0
- hexdag/builtin/adapters/mock/mock_database.py +225 -0
- hexdag/builtin/adapters/mock/mock_embedding.py +223 -0
- hexdag/builtin/adapters/mock/mock_llm.py +177 -0
- hexdag/builtin/adapters/mock/mock_tool_adapter.py +192 -0
- hexdag/builtin/adapters/mock/mock_tool_router.py +232 -0
- hexdag/builtin/adapters/openai/__init__.py +5 -0
- hexdag/builtin/adapters/openai/openai_adapter.py +634 -0
- hexdag/builtin/adapters/secret/__init__.py +7 -0
- hexdag/builtin/adapters/secret/local_secret_adapter.py +248 -0
- hexdag/builtin/adapters/unified_tool_router.py +280 -0
- hexdag/builtin/macros/__init__.py +17 -0
- hexdag/builtin/macros/conversation_agent.py +390 -0
- hexdag/builtin/macros/llm_macro.py +151 -0
- hexdag/builtin/macros/reasoning_agent.py +423 -0
- hexdag/builtin/macros/tool_macro.py +380 -0
- hexdag/builtin/nodes/__init__.py +38 -0
- hexdag/builtin/nodes/_discovery.py +123 -0
- hexdag/builtin/nodes/agent_node.py +696 -0
- hexdag/builtin/nodes/base_node_factory.py +242 -0
- hexdag/builtin/nodes/composite_node.py +926 -0
- hexdag/builtin/nodes/data_node.py +201 -0
- hexdag/builtin/nodes/expression_node.py +487 -0
- hexdag/builtin/nodes/function_node.py +454 -0
- hexdag/builtin/nodes/llm_node.py +491 -0
- hexdag/builtin/nodes/loop_node.py +920 -0
- hexdag/builtin/nodes/mapped_input.py +518 -0
- hexdag/builtin/nodes/port_call_node.py +269 -0
- hexdag/builtin/nodes/tool_call_node.py +195 -0
- hexdag/builtin/nodes/tool_utils.py +390 -0
- hexdag/builtin/prompts/__init__.py +68 -0
- hexdag/builtin/prompts/base.py +422 -0
- hexdag/builtin/prompts/chat_prompts.py +303 -0
- hexdag/builtin/prompts/error_correction_prompts.py +320 -0
- hexdag/builtin/prompts/tool_prompts.py +160 -0
- hexdag/builtin/tools/builtin_tools.py +84 -0
- hexdag/builtin/tools/database_tools.py +164 -0
- hexdag/cli/__init__.py +17 -0
- hexdag/cli/__main__.py +7 -0
- hexdag/cli/commands/__init__.py +27 -0
- hexdag/cli/commands/build_cmd.py +812 -0
- hexdag/cli/commands/create_cmd.py +208 -0
- hexdag/cli/commands/docs_cmd.py +293 -0
- hexdag/cli/commands/generate_types_cmd.py +252 -0
- hexdag/cli/commands/init_cmd.py +188 -0
- hexdag/cli/commands/pipeline_cmd.py +494 -0
- hexdag/cli/commands/plugin_dev_cmd.py +529 -0
- hexdag/cli/commands/plugins_cmd.py +441 -0
- hexdag/cli/commands/studio_cmd.py +101 -0
- hexdag/cli/commands/validate_cmd.py +221 -0
- hexdag/cli/main.py +84 -0
- hexdag/core/__init__.py +83 -0
- hexdag/core/config/__init__.py +20 -0
- hexdag/core/config/loader.py +479 -0
- hexdag/core/config/models.py +150 -0
- hexdag/core/configurable.py +294 -0
- hexdag/core/context/__init__.py +37 -0
- hexdag/core/context/execution_context.py +378 -0
- hexdag/core/docs/__init__.py +26 -0
- hexdag/core/docs/extractors.py +678 -0
- hexdag/core/docs/generators.py +890 -0
- hexdag/core/docs/models.py +120 -0
- hexdag/core/domain/__init__.py +10 -0
- hexdag/core/domain/dag.py +1225 -0
- hexdag/core/exceptions.py +234 -0
- hexdag/core/expression_parser.py +569 -0
- hexdag/core/logging.py +449 -0
- hexdag/core/models/__init__.py +17 -0
- hexdag/core/models/base.py +138 -0
- hexdag/core/orchestration/__init__.py +46 -0
- hexdag/core/orchestration/body_executor.py +481 -0
- hexdag/core/orchestration/components/__init__.py +97 -0
- hexdag/core/orchestration/components/adapter_lifecycle_manager.py +113 -0
- hexdag/core/orchestration/components/checkpoint_manager.py +134 -0
- hexdag/core/orchestration/components/execution_coordinator.py +360 -0
- hexdag/core/orchestration/components/health_check_manager.py +176 -0
- hexdag/core/orchestration/components/input_mapper.py +143 -0
- hexdag/core/orchestration/components/lifecycle_manager.py +583 -0
- hexdag/core/orchestration/components/node_executor.py +377 -0
- hexdag/core/orchestration/components/secret_manager.py +202 -0
- hexdag/core/orchestration/components/wave_executor.py +158 -0
- hexdag/core/orchestration/constants.py +17 -0
- hexdag/core/orchestration/events/README.md +312 -0
- hexdag/core/orchestration/events/__init__.py +104 -0
- hexdag/core/orchestration/events/batching.py +330 -0
- hexdag/core/orchestration/events/decorators.py +139 -0
- hexdag/core/orchestration/events/events.py +573 -0
- hexdag/core/orchestration/events/observers/__init__.py +30 -0
- hexdag/core/orchestration/events/observers/core_observers.py +690 -0
- hexdag/core/orchestration/events/observers/models.py +111 -0
- hexdag/core/orchestration/events/taxonomy.py +269 -0
- hexdag/core/orchestration/hook_context.py +237 -0
- hexdag/core/orchestration/hooks.py +437 -0
- hexdag/core/orchestration/models.py +418 -0
- hexdag/core/orchestration/orchestrator.py +910 -0
- hexdag/core/orchestration/orchestrator_factory.py +275 -0
- hexdag/core/orchestration/port_wrappers.py +327 -0
- hexdag/core/orchestration/prompt/__init__.py +32 -0
- hexdag/core/orchestration/prompt/template.py +332 -0
- hexdag/core/pipeline_builder/__init__.py +21 -0
- hexdag/core/pipeline_builder/component_instantiator.py +386 -0
- hexdag/core/pipeline_builder/include_tag.py +265 -0
- hexdag/core/pipeline_builder/pipeline_config.py +133 -0
- hexdag/core/pipeline_builder/py_tag.py +223 -0
- hexdag/core/pipeline_builder/tag_discovery.py +268 -0
- hexdag/core/pipeline_builder/yaml_builder.py +1196 -0
- hexdag/core/pipeline_builder/yaml_validator.py +569 -0
- hexdag/core/ports/__init__.py +65 -0
- hexdag/core/ports/api_call.py +133 -0
- hexdag/core/ports/database.py +489 -0
- hexdag/core/ports/embedding.py +215 -0
- hexdag/core/ports/executor.py +237 -0
- hexdag/core/ports/file_storage.py +117 -0
- hexdag/core/ports/healthcheck.py +87 -0
- hexdag/core/ports/llm.py +551 -0
- hexdag/core/ports/memory.py +70 -0
- hexdag/core/ports/observer_manager.py +130 -0
- hexdag/core/ports/secret.py +145 -0
- hexdag/core/ports/tool_router.py +94 -0
- hexdag/core/ports_builder.py +623 -0
- hexdag/core/protocols.py +273 -0
- hexdag/core/resolver.py +304 -0
- hexdag/core/schema/__init__.py +9 -0
- hexdag/core/schema/generator.py +742 -0
- hexdag/core/secrets.py +242 -0
- hexdag/core/types.py +413 -0
- hexdag/core/utils/async_warnings.py +206 -0
- hexdag/core/utils/schema_conversion.py +78 -0
- hexdag/core/utils/sql_validation.py +86 -0
- hexdag/core/validation/secure_json.py +148 -0
- hexdag/core/yaml_macro.py +517 -0
- hexdag/mcp_server.py +3120 -0
- hexdag/studio/__init__.py +10 -0
- hexdag/studio/build_ui.py +92 -0
- hexdag/studio/server/__init__.py +1 -0
- hexdag/studio/server/main.py +100 -0
- hexdag/studio/server/routes/__init__.py +9 -0
- hexdag/studio/server/routes/execute.py +208 -0
- hexdag/studio/server/routes/export.py +558 -0
- hexdag/studio/server/routes/files.py +207 -0
- hexdag/studio/server/routes/plugins.py +419 -0
- hexdag/studio/server/routes/validate.py +220 -0
- hexdag/studio/ui/index.html +13 -0
- hexdag/studio/ui/package-lock.json +2992 -0
- hexdag/studio/ui/package.json +31 -0
- hexdag/studio/ui/postcss.config.js +6 -0
- hexdag/studio/ui/public/hexdag.svg +5 -0
- hexdag/studio/ui/src/App.tsx +251 -0
- hexdag/studio/ui/src/components/Canvas.tsx +408 -0
- hexdag/studio/ui/src/components/ContextMenu.tsx +187 -0
- hexdag/studio/ui/src/components/FileBrowser.tsx +123 -0
- hexdag/studio/ui/src/components/Header.tsx +181 -0
- hexdag/studio/ui/src/components/HexdagNode.tsx +193 -0
- hexdag/studio/ui/src/components/NodeInspector.tsx +512 -0
- hexdag/studio/ui/src/components/NodePalette.tsx +262 -0
- hexdag/studio/ui/src/components/NodePortsSection.tsx +403 -0
- hexdag/studio/ui/src/components/PluginManager.tsx +347 -0
- hexdag/studio/ui/src/components/PortsEditor.tsx +481 -0
- hexdag/studio/ui/src/components/PythonEditor.tsx +195 -0
- hexdag/studio/ui/src/components/ValidationPanel.tsx +105 -0
- hexdag/studio/ui/src/components/YamlEditor.tsx +196 -0
- hexdag/studio/ui/src/components/index.ts +8 -0
- hexdag/studio/ui/src/index.css +92 -0
- hexdag/studio/ui/src/main.tsx +10 -0
- hexdag/studio/ui/src/types/index.ts +123 -0
- hexdag/studio/ui/src/vite-env.d.ts +1 -0
- hexdag/studio/ui/tailwind.config.js +29 -0
- hexdag/studio/ui/tsconfig.json +37 -0
- hexdag/studio/ui/tsconfig.node.json +13 -0
- hexdag/studio/ui/vite.config.ts +35 -0
- hexdag/visualization/__init__.py +69 -0
- hexdag/visualization/dag_visualizer.py +1020 -0
- hexdag-0.5.0.dev1.dist-info/METADATA +369 -0
- hexdag-0.5.0.dev1.dist-info/RECORD +261 -0
- hexdag-0.5.0.dev1.dist-info/WHEEL +4 -0
- hexdag-0.5.0.dev1.dist-info/entry_points.txt +4 -0
- hexdag-0.5.0.dev1.dist-info/licenses/LICENSE +190 -0
- hexdag_plugins/.gitignore +43 -0
- hexdag_plugins/README.md +73 -0
- hexdag_plugins/__init__.py +1 -0
- hexdag_plugins/azure/LICENSE +21 -0
- hexdag_plugins/azure/README.md +414 -0
- hexdag_plugins/azure/__init__.py +21 -0
- hexdag_plugins/azure/azure_blob_adapter.py +450 -0
- hexdag_plugins/azure/azure_cosmos_adapter.py +383 -0
- hexdag_plugins/azure/azure_keyvault_adapter.py +314 -0
- hexdag_plugins/azure/azure_openai_adapter.py +415 -0
- hexdag_plugins/azure/pyproject.toml +107 -0
- hexdag_plugins/azure/tests/__init__.py +1 -0
- hexdag_plugins/azure/tests/test_azure_blob_adapter.py +350 -0
- hexdag_plugins/azure/tests/test_azure_cosmos_adapter.py +323 -0
- hexdag_plugins/azure/tests/test_azure_keyvault_adapter.py +330 -0
- hexdag_plugins/azure/tests/test_azure_openai_adapter.py +329 -0
- hexdag_plugins/hexdag_etl/README.md +168 -0
- hexdag_plugins/hexdag_etl/__init__.py +53 -0
- hexdag_plugins/hexdag_etl/examples/01_simple_pandas_transform.py +270 -0
- hexdag_plugins/hexdag_etl/examples/02_simple_pandas_only.py +149 -0
- hexdag_plugins/hexdag_etl/examples/03_file_io_pipeline.py +109 -0
- hexdag_plugins/hexdag_etl/examples/test_pandas_transform.py +84 -0
- hexdag_plugins/hexdag_etl/hexdag.toml +25 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/__init__.py +48 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/__init__.py +13 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/api_extract.py +230 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/base_node_factory.py +181 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/file_io.py +415 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/outlook.py +492 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/pandas_transform.py +563 -0
- hexdag_plugins/hexdag_etl/hexdag_etl/nodes/sql_extract_load.py +112 -0
- hexdag_plugins/hexdag_etl/pyproject.toml +82 -0
- hexdag_plugins/hexdag_etl/test_transform.py +54 -0
- hexdag_plugins/hexdag_etl/tests/test_plugin_integration.py +62 -0
- hexdag_plugins/mysql_adapter/LICENSE +21 -0
- hexdag_plugins/mysql_adapter/README.md +224 -0
- hexdag_plugins/mysql_adapter/__init__.py +6 -0
- hexdag_plugins/mysql_adapter/mysql_adapter.py +408 -0
- hexdag_plugins/mysql_adapter/pyproject.toml +93 -0
- hexdag_plugins/mysql_adapter/tests/test_mysql_adapter.py +259 -0
- hexdag_plugins/storage/README.md +184 -0
- hexdag_plugins/storage/__init__.py +19 -0
- hexdag_plugins/storage/file/__init__.py +5 -0
- hexdag_plugins/storage/file/local.py +325 -0
- hexdag_plugins/storage/ports/__init__.py +5 -0
- hexdag_plugins/storage/ports/vector_store.py +236 -0
- hexdag_plugins/storage/sql/__init__.py +7 -0
- hexdag_plugins/storage/sql/base.py +187 -0
- hexdag_plugins/storage/sql/mysql.py +27 -0
- hexdag_plugins/storage/sql/postgresql.py +27 -0
- hexdag_plugins/storage/tests/__init__.py +1 -0
- hexdag_plugins/storage/tests/test_local_file_storage.py +161 -0
- hexdag_plugins/storage/tests/test_sql_adapters.py +212 -0
- hexdag_plugins/storage/vector/__init__.py +7 -0
- hexdag_plugins/storage/vector/chromadb.py +223 -0
- hexdag_plugins/storage/vector/in_memory.py +285 -0
- hexdag_plugins/storage/vector/pgvector.py +502 -0
|
@@ -0,0 +1,890 @@
|
|
|
1
|
+
"""Guide generators for MCP documentation.
|
|
2
|
+
|
|
3
|
+
This module generates documentation directly from extracted component
|
|
4
|
+
documentation - no external templates needed.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from hexdag.core.docs.models import AdapterDoc, NodeDoc, ToolDoc
|
|
12
|
+
from hexdag.core.logging import get_logger
|
|
13
|
+
|
|
14
|
+
logger = get_logger(__name__)
|
|
15
|
+
|
|
16
|
+
# Path to the generated schema
|
|
17
|
+
SCHEMA_PATH = Path(__file__).parent.parent.parent.parent / "schemas" / "pipeline-schema.json"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class GuideGenerator:
|
|
21
|
+
"""Generate documentation guides from extracted component docs.
|
|
22
|
+
|
|
23
|
+
All documentation is generated programmatically from code introspection,
|
|
24
|
+
ensuring it stays in sync with the actual implementation.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def generate_adapter_guide(self, adapters: list[AdapterDoc]) -> str:
|
|
28
|
+
"""Generate adapter creation guide.
|
|
29
|
+
|
|
30
|
+
Parameters
|
|
31
|
+
----------
|
|
32
|
+
adapters : list[AdapterDoc]
|
|
33
|
+
List of adapter documentation objects
|
|
34
|
+
|
|
35
|
+
Returns
|
|
36
|
+
-------
|
|
37
|
+
str
|
|
38
|
+
Complete adapter guide as markdown
|
|
39
|
+
"""
|
|
40
|
+
# Group adapters by port type
|
|
41
|
+
adapters_by_port: dict[str, list[AdapterDoc]] = {}
|
|
42
|
+
for adapter in adapters:
|
|
43
|
+
port = adapter.port_type
|
|
44
|
+
if port not in adapters_by_port:
|
|
45
|
+
adapters_by_port[port] = []
|
|
46
|
+
adapters_by_port[port].append(adapter)
|
|
47
|
+
|
|
48
|
+
lines = [
|
|
49
|
+
"# Creating Custom Adapters in hexDAG",
|
|
50
|
+
"",
|
|
51
|
+
"## Overview",
|
|
52
|
+
"",
|
|
53
|
+
"hexDAG uses adapters to connect pipelines to external services like LLMs,",
|
|
54
|
+
"databases, and APIs. Adapters implement ports (interfaces) with async methods.",
|
|
55
|
+
"",
|
|
56
|
+
"## Quick Start",
|
|
57
|
+
"",
|
|
58
|
+
"### Simple Adapter (No Secrets)",
|
|
59
|
+
"",
|
|
60
|
+
"```python",
|
|
61
|
+
"class MemoryCacheAdapter:",
|
|
62
|
+
' """Simple in-memory cache adapter."""',
|
|
63
|
+
"",
|
|
64
|
+
" def __init__(self, max_size: int = 100, ttl: int = 3600):",
|
|
65
|
+
" self.cache = {}",
|
|
66
|
+
" self.max_size = max_size",
|
|
67
|
+
" self.ttl = ttl",
|
|
68
|
+
"",
|
|
69
|
+
" async def aget(self, key: str):",
|
|
70
|
+
" return self.cache.get(key)",
|
|
71
|
+
"",
|
|
72
|
+
" async def aset(self, key: str, value: any):",
|
|
73
|
+
" self.cache[key] = value",
|
|
74
|
+
"```",
|
|
75
|
+
"",
|
|
76
|
+
"### Adapter with Secrets",
|
|
77
|
+
"",
|
|
78
|
+
"Use `secret()` in defaults to declare secrets that auto-resolve from environment:",
|
|
79
|
+
"",
|
|
80
|
+
"```python",
|
|
81
|
+
"from hexdag.core.secrets import secret",
|
|
82
|
+
"",
|
|
83
|
+
"class OpenAIAdapter:",
|
|
84
|
+
' """OpenAI LLM adapter with automatic secret resolution."""',
|
|
85
|
+
"",
|
|
86
|
+
" def __init__(",
|
|
87
|
+
" self,",
|
|
88
|
+
' api_key: str = secret(env="OPENAI_API_KEY"), # Auto-resolved',
|
|
89
|
+
' model: str = "gpt-4",',
|
|
90
|
+
" temperature: float = 0.7",
|
|
91
|
+
" ):",
|
|
92
|
+
" self.api_key = api_key",
|
|
93
|
+
" self.model = model",
|
|
94
|
+
" self.temperature = temperature",
|
|
95
|
+
"",
|
|
96
|
+
" async def aresponse(self, messages: list) -> str:",
|
|
97
|
+
" # Your implementation using self.api_key",
|
|
98
|
+
" ...",
|
|
99
|
+
"```",
|
|
100
|
+
"",
|
|
101
|
+
"## Secret Resolution",
|
|
102
|
+
"",
|
|
103
|
+
"Secrets declared with `secret()` are resolved in this order:",
|
|
104
|
+
"1. **Explicit kwargs** - Values passed directly to `__init__`",
|
|
105
|
+
"2. **Environment variables** - From the `env` parameter",
|
|
106
|
+
"3. **Memory port** - From orchestrator memory (with `secret:` prefix)",
|
|
107
|
+
"4. **Error** - If required and no default",
|
|
108
|
+
"",
|
|
109
|
+
"## Available Adapters",
|
|
110
|
+
"",
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
# Generate adapter tables by port type
|
|
114
|
+
for port_type in sorted(adapters_by_port.keys()):
|
|
115
|
+
port_adapters = adapters_by_port[port_type]
|
|
116
|
+
lines.append(f"### {port_type}")
|
|
117
|
+
lines.append("")
|
|
118
|
+
lines.append("| Adapter | Description |")
|
|
119
|
+
lines.append("|---------|-------------|")
|
|
120
|
+
for adapter in port_adapters:
|
|
121
|
+
desc = adapter.description[:60]
|
|
122
|
+
if len(adapter.description) > 60:
|
|
123
|
+
desc += "..."
|
|
124
|
+
lines.append(f"| `{adapter.name}` | {desc} |")
|
|
125
|
+
lines.append("")
|
|
126
|
+
|
|
127
|
+
# Add YAML usage section
|
|
128
|
+
lines.extend([
|
|
129
|
+
"## Using Adapters in YAML",
|
|
130
|
+
"",
|
|
131
|
+
"```yaml",
|
|
132
|
+
"apiVersion: hexdag/v1",
|
|
133
|
+
"kind: Pipeline",
|
|
134
|
+
"metadata:",
|
|
135
|
+
" name: my-pipeline",
|
|
136
|
+
"spec:",
|
|
137
|
+
" ports:",
|
|
138
|
+
" llm:",
|
|
139
|
+
" adapter: hexdag.builtin.adapters.openai.OpenAIAdapter",
|
|
140
|
+
" config:",
|
|
141
|
+
" api_key: ${OPENAI_API_KEY}",
|
|
142
|
+
" model: gpt-4",
|
|
143
|
+
"",
|
|
144
|
+
" nodes:",
|
|
145
|
+
" - kind: llm_node",
|
|
146
|
+
" metadata:",
|
|
147
|
+
" name: analyzer",
|
|
148
|
+
" spec:",
|
|
149
|
+
' prompt_template: "Analyze: {{input}}"',
|
|
150
|
+
" dependencies: []",
|
|
151
|
+
"```",
|
|
152
|
+
"",
|
|
153
|
+
"## Best Practices",
|
|
154
|
+
"",
|
|
155
|
+
"1. **Async First**: Use `async def` for I/O operations",
|
|
156
|
+
"2. **Type Hints**: Add type annotations for better tooling",
|
|
157
|
+
"3. **Docstrings**: Document your adapter's purpose and config",
|
|
158
|
+
"4. **Error Handling**: Wrap external calls in try/except",
|
|
159
|
+
"5. **Secrets**: Use `secret()` - never hardcode secrets",
|
|
160
|
+
])
|
|
161
|
+
|
|
162
|
+
return "\n".join(lines)
|
|
163
|
+
|
|
164
|
+
def generate_node_guide(self, nodes: list[NodeDoc]) -> str:
|
|
165
|
+
"""Generate node creation guide.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
nodes : list[NodeDoc]
|
|
170
|
+
List of node documentation objects
|
|
171
|
+
|
|
172
|
+
Returns
|
|
173
|
+
-------
|
|
174
|
+
str
|
|
175
|
+
Complete node guide as markdown
|
|
176
|
+
"""
|
|
177
|
+
lines = [
|
|
178
|
+
"# Creating Custom Nodes in hexDAG",
|
|
179
|
+
"",
|
|
180
|
+
"## Overview",
|
|
181
|
+
"",
|
|
182
|
+
"Nodes are the building blocks of hexDAG pipelines. Each node performs a specific",
|
|
183
|
+
"task and can be connected to other nodes via dependencies.",
|
|
184
|
+
"",
|
|
185
|
+
"## Quick Start",
|
|
186
|
+
"",
|
|
187
|
+
"### Using FunctionNode (Simplest)",
|
|
188
|
+
"",
|
|
189
|
+
"Reference any Python function by module path:",
|
|
190
|
+
"",
|
|
191
|
+
"```yaml",
|
|
192
|
+
"- kind: function_node",
|
|
193
|
+
" metadata:",
|
|
194
|
+
" name: my_processor",
|
|
195
|
+
" spec:",
|
|
196
|
+
" fn: mycompany.processors.process_data",
|
|
197
|
+
" dependencies: []",
|
|
198
|
+
"```",
|
|
199
|
+
"",
|
|
200
|
+
"```python",
|
|
201
|
+
"# mycompany/processors.py",
|
|
202
|
+
"def process_data(input_data: dict) -> dict:",
|
|
203
|
+
' """Your processing logic."""',
|
|
204
|
+
' return {"result": input_data["value"] * 2}',
|
|
205
|
+
"```",
|
|
206
|
+
"",
|
|
207
|
+
"## Available Node Types",
|
|
208
|
+
"",
|
|
209
|
+
]
|
|
210
|
+
|
|
211
|
+
# Generate node documentation
|
|
212
|
+
for node in nodes:
|
|
213
|
+
lines.append(f"### {node.name}")
|
|
214
|
+
lines.append("")
|
|
215
|
+
lines.append(f"{node.description}")
|
|
216
|
+
lines.append("")
|
|
217
|
+
lines.append(f"**Kind**: `{node.kind}`")
|
|
218
|
+
lines.append("")
|
|
219
|
+
|
|
220
|
+
if node.parameters:
|
|
221
|
+
lines.append("| Parameter | Type | Required | Description |")
|
|
222
|
+
lines.append("|-----------|------|----------|-------------|")
|
|
223
|
+
for param in node.parameters:
|
|
224
|
+
req = "Yes" if param.required else "No"
|
|
225
|
+
desc = param.description[:50]
|
|
226
|
+
if len(param.description) > 50:
|
|
227
|
+
desc += "..."
|
|
228
|
+
lines.append(f"| `{param.name}` | `{param.type_hint}` | {req} | {desc} |")
|
|
229
|
+
lines.append("")
|
|
230
|
+
|
|
231
|
+
if node.yaml_example:
|
|
232
|
+
lines.append("**Example:**")
|
|
233
|
+
lines.append("```yaml")
|
|
234
|
+
lines.append(node.yaml_example.strip())
|
|
235
|
+
lines.append("```")
|
|
236
|
+
lines.append("")
|
|
237
|
+
|
|
238
|
+
# Add custom node section
|
|
239
|
+
lines.extend([
|
|
240
|
+
"## Creating Custom Nodes",
|
|
241
|
+
"",
|
|
242
|
+
"```python",
|
|
243
|
+
"from hexdag.builtin.nodes import BaseNodeFactory",
|
|
244
|
+
"from hexdag.core.domain.dag import NodeSpec",
|
|
245
|
+
"",
|
|
246
|
+
"class CustomProcessorNode(BaseNodeFactory):",
|
|
247
|
+
' """Custom node for specialized processing."""',
|
|
248
|
+
"",
|
|
249
|
+
" def __call__(",
|
|
250
|
+
" self,",
|
|
251
|
+
" name: str,",
|
|
252
|
+
" threshold: float = 0.5,",
|
|
253
|
+
" **kwargs",
|
|
254
|
+
" ) -> NodeSpec:",
|
|
255
|
+
" async def process_fn(input_data: dict) -> dict:",
|
|
256
|
+
' if input_data.get("score", 0) > threshold:',
|
|
257
|
+
' return {"status": "pass"}',
|
|
258
|
+
' return {"status": "fail"}',
|
|
259
|
+
"",
|
|
260
|
+
" return NodeSpec(",
|
|
261
|
+
" name=name,",
|
|
262
|
+
" fn=process_fn,",
|
|
263
|
+
' deps=frozenset(kwargs.get("deps", [])),',
|
|
264
|
+
" )",
|
|
265
|
+
"```",
|
|
266
|
+
"",
|
|
267
|
+
"## Best Practices",
|
|
268
|
+
"",
|
|
269
|
+
"1. **Async Functions**: Use `async def` for the node function",
|
|
270
|
+
"2. **Immutable**: Don't modify input_data; return new dict",
|
|
271
|
+
"3. **Type Hints**: Add types for better IDE support",
|
|
272
|
+
"4. **Docstrings**: Document purpose and parameters",
|
|
273
|
+
])
|
|
274
|
+
|
|
275
|
+
return "\n".join(lines)
|
|
276
|
+
|
|
277
|
+
def generate_tool_guide(self, tools: list[ToolDoc]) -> str:
|
|
278
|
+
"""Generate tool creation guide.
|
|
279
|
+
|
|
280
|
+
Parameters
|
|
281
|
+
----------
|
|
282
|
+
tools : list[ToolDoc]
|
|
283
|
+
List of tool documentation objects
|
|
284
|
+
|
|
285
|
+
Returns
|
|
286
|
+
-------
|
|
287
|
+
str
|
|
288
|
+
Complete tool guide as markdown
|
|
289
|
+
"""
|
|
290
|
+
# Separate sync and async tools
|
|
291
|
+
async_tools = [t for t in tools if t.is_async]
|
|
292
|
+
sync_tools = [t for t in tools if not t.is_async]
|
|
293
|
+
|
|
294
|
+
lines = [
|
|
295
|
+
"# Creating Custom Tools for hexDAG Agents",
|
|
296
|
+
"",
|
|
297
|
+
"## Overview",
|
|
298
|
+
"",
|
|
299
|
+
"Tools are functions that agents can invoke during execution. They enable",
|
|
300
|
+
"agents to interact with external systems, perform calculations, or access data.",
|
|
301
|
+
"",
|
|
302
|
+
"## Quick Start",
|
|
303
|
+
"",
|
|
304
|
+
"```python",
|
|
305
|
+
"def calculate(expression: str) -> str:",
|
|
306
|
+
' """Evaluate a mathematical expression.',
|
|
307
|
+
"",
|
|
308
|
+
" Args:",
|
|
309
|
+
' expression: Math expression like "2 + 2"',
|
|
310
|
+
"",
|
|
311
|
+
" Returns:",
|
|
312
|
+
" Result as a string",
|
|
313
|
+
' """',
|
|
314
|
+
" result = eval(expression) # Use safe evaluation in production",
|
|
315
|
+
" return str(result)",
|
|
316
|
+
"```",
|
|
317
|
+
"",
|
|
318
|
+
"## Built-in Tools",
|
|
319
|
+
"",
|
|
320
|
+
]
|
|
321
|
+
|
|
322
|
+
# Generate tool documentation
|
|
323
|
+
for tool in tools:
|
|
324
|
+
lines.append(f"### {tool.name}")
|
|
325
|
+
lines.append("")
|
|
326
|
+
lines.append(tool.description)
|
|
327
|
+
lines.append("")
|
|
328
|
+
|
|
329
|
+
if tool.parameters:
|
|
330
|
+
lines.append("**Parameters:**")
|
|
331
|
+
for param in tool.parameters:
|
|
332
|
+
opt = "" if param.required else ", optional"
|
|
333
|
+
default = f" Default: `{param.default}`" if param.default else ""
|
|
334
|
+
lines.append(
|
|
335
|
+
f"- `{param.name}` (`{param.type_hint}`{opt}): {param.description}{default}"
|
|
336
|
+
)
|
|
337
|
+
lines.append("")
|
|
338
|
+
|
|
339
|
+
lines.append(f"**Returns:** `{tool.return_type}`")
|
|
340
|
+
if tool.is_async:
|
|
341
|
+
lines.append("")
|
|
342
|
+
lines.append("*This is an async tool.*")
|
|
343
|
+
lines.append("")
|
|
344
|
+
|
|
345
|
+
# Add usage section
|
|
346
|
+
lines.extend([
|
|
347
|
+
"## Using Tools with Agents",
|
|
348
|
+
"",
|
|
349
|
+
"```yaml",
|
|
350
|
+
"- kind: agent_node",
|
|
351
|
+
" metadata:",
|
|
352
|
+
" name: research_agent",
|
|
353
|
+
" spec:",
|
|
354
|
+
' initial_prompt_template: "Research: {{topic}}"',
|
|
355
|
+
" max_steps: 5",
|
|
356
|
+
" tools:",
|
|
357
|
+
" - hexdag.builtin.tools.builtin_tools.tool_end",
|
|
358
|
+
" - mycompany.tools.search",
|
|
359
|
+
" dependencies: []",
|
|
360
|
+
"```",
|
|
361
|
+
"",
|
|
362
|
+
"## Tool Invocation Format",
|
|
363
|
+
"",
|
|
364
|
+
"Agents invoke tools using:",
|
|
365
|
+
"```",
|
|
366
|
+
'INVOKE_TOOL: tool_name(param1="value", param2=123)',
|
|
367
|
+
"```",
|
|
368
|
+
"",
|
|
369
|
+
"## Tool Reference",
|
|
370
|
+
"",
|
|
371
|
+
])
|
|
372
|
+
|
|
373
|
+
# Add reference tables
|
|
374
|
+
if sync_tools:
|
|
375
|
+
lines.append("### Synchronous Tools")
|
|
376
|
+
lines.append("")
|
|
377
|
+
lines.append("| Tool | Description | Return Type |")
|
|
378
|
+
lines.append("|------|-------------|-------------|")
|
|
379
|
+
for tool in sync_tools:
|
|
380
|
+
desc = tool.description[:40]
|
|
381
|
+
if len(tool.description) > 40:
|
|
382
|
+
desc += "..."
|
|
383
|
+
lines.append(f"| `{tool.name}` | {desc} | `{tool.return_type}` |")
|
|
384
|
+
lines.append("")
|
|
385
|
+
|
|
386
|
+
if async_tools:
|
|
387
|
+
lines.append("### Asynchronous Tools")
|
|
388
|
+
lines.append("")
|
|
389
|
+
lines.append("| Tool | Description | Return Type |")
|
|
390
|
+
lines.append("|------|-------------|-------------|")
|
|
391
|
+
for tool in async_tools:
|
|
392
|
+
desc = tool.description[:40]
|
|
393
|
+
if len(tool.description) > 40:
|
|
394
|
+
desc += "..."
|
|
395
|
+
lines.append(f"| `{tool.name}` | {desc} | `{tool.return_type}` |")
|
|
396
|
+
lines.append("")
|
|
397
|
+
|
|
398
|
+
lines.extend([
|
|
399
|
+
"## Best Practices",
|
|
400
|
+
"",
|
|
401
|
+
"1. **Type Hints**: Always add parameter and return types",
|
|
402
|
+
"2. **Docstrings**: Write clear descriptions for LLM understanding",
|
|
403
|
+
"3. **Error Handling**: Return error messages, don't raise exceptions",
|
|
404
|
+
"4. **Idempotent**: Tools should be safe to retry",
|
|
405
|
+
])
|
|
406
|
+
|
|
407
|
+
return "\n".join(lines)
|
|
408
|
+
|
|
409
|
+
def generate_syntax_reference(self) -> str:
|
|
410
|
+
"""Generate syntax reference guide.
|
|
411
|
+
|
|
412
|
+
Returns
|
|
413
|
+
-------
|
|
414
|
+
str
|
|
415
|
+
Complete syntax reference as markdown
|
|
416
|
+
"""
|
|
417
|
+
return """# hexDAG Variable Reference Syntax
|
|
418
|
+
|
|
419
|
+
## 1. Initial Input Reference: $input
|
|
420
|
+
|
|
421
|
+
Use `$input.field` in `input_mapping` to access the original pipeline input.
|
|
422
|
+
|
|
423
|
+
```yaml
|
|
424
|
+
nodes:
|
|
425
|
+
- kind: function_node
|
|
426
|
+
metadata:
|
|
427
|
+
name: processor
|
|
428
|
+
spec:
|
|
429
|
+
fn: myapp.process
|
|
430
|
+
input_mapping:
|
|
431
|
+
load_id: $input.load_id
|
|
432
|
+
carrier: $input.carrier_mc
|
|
433
|
+
dependencies: [extractor]
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
## 2. Node Output Reference: {{node.field}}
|
|
437
|
+
|
|
438
|
+
Use Jinja2 syntax in prompt templates to reference previous node outputs.
|
|
439
|
+
|
|
440
|
+
```yaml
|
|
441
|
+
- kind: llm_node
|
|
442
|
+
metadata:
|
|
443
|
+
name: analyzer
|
|
444
|
+
spec:
|
|
445
|
+
prompt_template: |
|
|
446
|
+
Analyze this data:
|
|
447
|
+
{{extractor.result}}
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
## 3. Environment Variables: ${VAR}
|
|
451
|
+
|
|
452
|
+
```yaml
|
|
453
|
+
spec:
|
|
454
|
+
ports:
|
|
455
|
+
llm:
|
|
456
|
+
config:
|
|
457
|
+
model: ${MODEL} # Resolved at build time
|
|
458
|
+
api_key: ${OPENAI_API_KEY} # Secret - resolved at runtime
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
**Secret Patterns (deferred to runtime):**
|
|
462
|
+
- `*_API_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIAL`, `SECRET_*`
|
|
463
|
+
|
|
464
|
+
## 4. Input Mapping
|
|
465
|
+
|
|
466
|
+
```yaml
|
|
467
|
+
- kind: function_node
|
|
468
|
+
metadata:
|
|
469
|
+
name: merger
|
|
470
|
+
spec:
|
|
471
|
+
fn: myapp.merge_results
|
|
472
|
+
input_mapping:
|
|
473
|
+
request_id: $input.id # From initial input
|
|
474
|
+
analysis: analyzer.result # From dependency
|
|
475
|
+
dependencies: [analyzer]
|
|
476
|
+
```
|
|
477
|
+
|
|
478
|
+
## Quick Reference
|
|
479
|
+
|
|
480
|
+
| Syntax | Location | Purpose |
|
|
481
|
+
|--------|----------|---------|
|
|
482
|
+
| `$input.field` | input_mapping | Access initial pipeline input |
|
|
483
|
+
| `{{node.field}}` | prompt_template | Jinja2 template reference |
|
|
484
|
+
| `${VAR}` | Any string | Environment variable |
|
|
485
|
+
| `${VAR:default}` | Any string | Env var with default |
|
|
486
|
+
| `node.path` | input_mapping | Dependency output extraction |
|
|
487
|
+
"""
|
|
488
|
+
|
|
489
|
+
def generate_extension_guide(
|
|
490
|
+
self,
|
|
491
|
+
adapters: list[AdapterDoc],
|
|
492
|
+
nodes: list[NodeDoc],
|
|
493
|
+
tools: list[ToolDoc],
|
|
494
|
+
) -> str:
|
|
495
|
+
"""Generate extension overview guide.
|
|
496
|
+
|
|
497
|
+
Parameters
|
|
498
|
+
----------
|
|
499
|
+
adapters : list[AdapterDoc]
|
|
500
|
+
List of adapter documentation objects
|
|
501
|
+
nodes : list[NodeDoc]
|
|
502
|
+
List of node documentation objects
|
|
503
|
+
tools : list[ToolDoc]
|
|
504
|
+
List of tool documentation objects
|
|
505
|
+
|
|
506
|
+
Returns
|
|
507
|
+
-------
|
|
508
|
+
str
|
|
509
|
+
Complete extension guide as markdown
|
|
510
|
+
"""
|
|
511
|
+
lines = [
|
|
512
|
+
"# Extending hexDAG - Overview",
|
|
513
|
+
"",
|
|
514
|
+
"## Extension Points",
|
|
515
|
+
"",
|
|
516
|
+
"| Component | Purpose | Available |",
|
|
517
|
+
"|-----------|---------|-----------|",
|
|
518
|
+
f"| **Adapter** | Connect to external services | {len(adapters)} |",
|
|
519
|
+
f"| **Node** | Custom processing logic | {len(nodes)} |",
|
|
520
|
+
f"| **Tool** | Agent-callable functions | {len(tools)} |",
|
|
521
|
+
"",
|
|
522
|
+
"## Quick Reference",
|
|
523
|
+
"",
|
|
524
|
+
"### Adapters",
|
|
525
|
+
"Use `get_custom_adapter_guide()` for full documentation.",
|
|
526
|
+
"",
|
|
527
|
+
"### Nodes",
|
|
528
|
+
"Use `get_custom_node_guide()` for full documentation.",
|
|
529
|
+
"",
|
|
530
|
+
"### Tools",
|
|
531
|
+
"Use `get_custom_tool_guide()` for full documentation.",
|
|
532
|
+
"",
|
|
533
|
+
"## MCP Tools for Development",
|
|
534
|
+
"",
|
|
535
|
+
"| Tool | Purpose |",
|
|
536
|
+
"|------|---------|",
|
|
537
|
+
"| `list_nodes()` | See available nodes |",
|
|
538
|
+
"| `list_adapters()` | See available adapters |",
|
|
539
|
+
"| `list_tools()` | See available tools |",
|
|
540
|
+
"| `get_component_schema()` | Get config schema |",
|
|
541
|
+
"| `validate_yaml_pipeline()` | Validate your YAML |",
|
|
542
|
+
"| `get_pipeline_schema()` | Get full JSON schema |",
|
|
543
|
+
]
|
|
544
|
+
|
|
545
|
+
return "\n".join(lines)
|
|
546
|
+
|
|
547
|
+
def generate_pipeline_schema_guide(self) -> str:
|
|
548
|
+
"""Generate pipeline schema reference guide from JSON schema.
|
|
549
|
+
|
|
550
|
+
Reads the auto-generated pipeline-schema.json and produces
|
|
551
|
+
a human-readable markdown reference.
|
|
552
|
+
|
|
553
|
+
Returns
|
|
554
|
+
-------
|
|
555
|
+
str
|
|
556
|
+
Complete pipeline schema guide as markdown
|
|
557
|
+
"""
|
|
558
|
+
lines = [
|
|
559
|
+
"# hexDAG Pipeline Schema Reference",
|
|
560
|
+
"",
|
|
561
|
+
"This reference is auto-generated from the pipeline JSON schema.",
|
|
562
|
+
"",
|
|
563
|
+
"## Overview",
|
|
564
|
+
"",
|
|
565
|
+
"hexDAG pipelines are defined in YAML using a Kubernetes-like structure.",
|
|
566
|
+
"The schema provides validation and IDE autocompletion support.",
|
|
567
|
+
"",
|
|
568
|
+
"## Pipeline Structure",
|
|
569
|
+
"",
|
|
570
|
+
"```yaml",
|
|
571
|
+
"apiVersion: hexdag/v1",
|
|
572
|
+
"kind: Pipeline",
|
|
573
|
+
"metadata:",
|
|
574
|
+
" name: my-pipeline",
|
|
575
|
+
" description: Pipeline description",
|
|
576
|
+
"spec:",
|
|
577
|
+
" ports: {} # Adapter configurations",
|
|
578
|
+
" nodes: [] # Processing nodes",
|
|
579
|
+
" events: {} # Event handlers",
|
|
580
|
+
"```",
|
|
581
|
+
"",
|
|
582
|
+
]
|
|
583
|
+
|
|
584
|
+
# Try to load and parse schema
|
|
585
|
+
try:
|
|
586
|
+
schema = self._load_schema()
|
|
587
|
+
if schema:
|
|
588
|
+
lines.extend(self._generate_node_types_section(schema))
|
|
589
|
+
lines.extend(self._generate_ports_section(schema))
|
|
590
|
+
lines.extend(self._generate_events_section(schema))
|
|
591
|
+
except Exception as e:
|
|
592
|
+
logger.warning(f"Could not load pipeline schema: {e}")
|
|
593
|
+
lines.extend([
|
|
594
|
+
"## Node Types",
|
|
595
|
+
"",
|
|
596
|
+
"*Schema not available. Run `scripts/generate_schemas.py` first.*",
|
|
597
|
+
"",
|
|
598
|
+
])
|
|
599
|
+
|
|
600
|
+
# Add IDE setup section
|
|
601
|
+
lines.extend([
|
|
602
|
+
"## IDE Setup",
|
|
603
|
+
"",
|
|
604
|
+
"### VS Code",
|
|
605
|
+
"",
|
|
606
|
+
"Add to `.vscode/settings.json`:",
|
|
607
|
+
"",
|
|
608
|
+
"```json",
|
|
609
|
+
"{",
|
|
610
|
+
' "yaml.schemas": {',
|
|
611
|
+
' "./schemas/pipeline-schema.json": ["*.yaml", "pipelines/*.yaml"]',
|
|
612
|
+
" }",
|
|
613
|
+
"}",
|
|
614
|
+
"```",
|
|
615
|
+
"",
|
|
616
|
+
"### Schema Location",
|
|
617
|
+
"",
|
|
618
|
+
(
|
|
619
|
+
"The schema file is at `schemas/pipeline-schema.json` and is "
|
|
620
|
+
"auto-generated from node `_yaml_schema` attributes."
|
|
621
|
+
),
|
|
622
|
+
])
|
|
623
|
+
|
|
624
|
+
return "\n".join(lines)
|
|
625
|
+
|
|
626
|
+
def _load_schema(self) -> dict[str, Any] | None:
|
|
627
|
+
"""Load the pipeline schema JSON file.
|
|
628
|
+
|
|
629
|
+
Returns
|
|
630
|
+
-------
|
|
631
|
+
dict[str, Any] | None
|
|
632
|
+
Parsed schema or None if not found
|
|
633
|
+
"""
|
|
634
|
+
if not SCHEMA_PATH.exists():
|
|
635
|
+
return None
|
|
636
|
+
return json.loads(SCHEMA_PATH.read_text())
|
|
637
|
+
|
|
638
|
+
def _generate_node_types_section(self, schema: dict[str, Any]) -> list[str]:
|
|
639
|
+
"""Generate node types documentation from schema.
|
|
640
|
+
|
|
641
|
+
Parameters
|
|
642
|
+
----------
|
|
643
|
+
schema : dict[str, Any]
|
|
644
|
+
Parsed JSON schema
|
|
645
|
+
|
|
646
|
+
Returns
|
|
647
|
+
-------
|
|
648
|
+
list[str]
|
|
649
|
+
Lines of markdown documentation
|
|
650
|
+
"""
|
|
651
|
+
lines = ["## Node Types", ""]
|
|
652
|
+
|
|
653
|
+
defs = schema.get("$defs", {})
|
|
654
|
+
|
|
655
|
+
# Find all node specs
|
|
656
|
+
node_specs = [(name, spec) for name, spec in defs.items() if name.endswith("NodeSpec")]
|
|
657
|
+
|
|
658
|
+
if not node_specs:
|
|
659
|
+
lines.append("*No node types found in schema.*")
|
|
660
|
+
lines.append("")
|
|
661
|
+
return lines
|
|
662
|
+
|
|
663
|
+
# Generate table
|
|
664
|
+
lines.append("| Node Kind | Description |")
|
|
665
|
+
lines.append("|-----------|-------------|")
|
|
666
|
+
|
|
667
|
+
for name, spec in sorted(node_specs):
|
|
668
|
+
kind = name.replace("NodeSpec", "").lower()
|
|
669
|
+
# Convert CamelCase to snake_case
|
|
670
|
+
kind = "".join(f"_{c.lower()}" if c.isupper() else c for c in kind).lstrip("_")
|
|
671
|
+
|
|
672
|
+
# Get description from spec
|
|
673
|
+
desc = spec.get("description", "")
|
|
674
|
+
if not desc:
|
|
675
|
+
props = spec.get("properties", {})
|
|
676
|
+
spec_prop = props.get("spec", {})
|
|
677
|
+
desc = spec_prop.get("description", "No description")
|
|
678
|
+
|
|
679
|
+
# Truncate long descriptions
|
|
680
|
+
if len(desc) > 60:
|
|
681
|
+
desc = desc[:57] + "..."
|
|
682
|
+
|
|
683
|
+
lines.append(f"| `{kind}_node` | {desc} |")
|
|
684
|
+
|
|
685
|
+
lines.append("")
|
|
686
|
+
|
|
687
|
+
# Generate detailed sections for each node
|
|
688
|
+
for name, spec in sorted(node_specs):
|
|
689
|
+
lines.extend(self._generate_node_detail(name, spec))
|
|
690
|
+
|
|
691
|
+
return lines
|
|
692
|
+
|
|
693
|
+
def _generate_node_detail(self, name: str, spec: dict[str, Any]) -> list[str]:
|
|
694
|
+
"""Generate detailed documentation for a single node type.
|
|
695
|
+
|
|
696
|
+
Parameters
|
|
697
|
+
----------
|
|
698
|
+
name : str
|
|
699
|
+
Node spec name (e.g., "FunctionNodeSpec")
|
|
700
|
+
spec : dict[str, Any]
|
|
701
|
+
Node specification from schema
|
|
702
|
+
|
|
703
|
+
Returns
|
|
704
|
+
-------
|
|
705
|
+
list[str]
|
|
706
|
+
Lines of markdown documentation
|
|
707
|
+
"""
|
|
708
|
+
kind = name.replace("NodeSpec", "").lower()
|
|
709
|
+
kind = "".join(f"_{c.lower()}" if c.isupper() else c for c in kind).lstrip("_")
|
|
710
|
+
|
|
711
|
+
lines = [f"### {kind}_node", ""]
|
|
712
|
+
|
|
713
|
+
# Get description
|
|
714
|
+
props = spec.get("properties", {})
|
|
715
|
+
spec_prop = props.get("spec", {})
|
|
716
|
+
desc = spec_prop.get("description", spec.get("description", ""))
|
|
717
|
+
if desc:
|
|
718
|
+
lines.append(desc)
|
|
719
|
+
lines.append("")
|
|
720
|
+
|
|
721
|
+
# Extract spec properties
|
|
722
|
+
spec_props = spec_prop.get("properties", {})
|
|
723
|
+
required = spec_prop.get("required", [])
|
|
724
|
+
|
|
725
|
+
if spec_props:
|
|
726
|
+
lines.append("**Parameters:**")
|
|
727
|
+
lines.append("")
|
|
728
|
+
lines.append("| Parameter | Type | Required | Description |")
|
|
729
|
+
lines.append("|-----------|------|----------|-------------|")
|
|
730
|
+
|
|
731
|
+
for param_name, param_spec in sorted(spec_props.items()):
|
|
732
|
+
param_type = self._get_type_from_schema(param_spec)
|
|
733
|
+
is_required = "Yes" if param_name in required else "No"
|
|
734
|
+
param_desc = param_spec.get("description", "")
|
|
735
|
+
if len(param_desc) > 40:
|
|
736
|
+
param_desc = param_desc[:37] + "..."
|
|
737
|
+
lines.append(f"| `{param_name}` | {param_type} | {is_required} | {param_desc} |")
|
|
738
|
+
|
|
739
|
+
lines.append("")
|
|
740
|
+
|
|
741
|
+
# Add example
|
|
742
|
+
lines.append("**Example:**")
|
|
743
|
+
lines.append("")
|
|
744
|
+
lines.append("```yaml")
|
|
745
|
+
lines.append(f"- kind: {kind}_node")
|
|
746
|
+
lines.append(" metadata:")
|
|
747
|
+
lines.append(f" name: my_{kind}")
|
|
748
|
+
lines.append(" spec:")
|
|
749
|
+
|
|
750
|
+
# Add required params as example (limit to first 3)
|
|
751
|
+
lines.extend(f" {param_name}: # required" for param_name in required[:3])
|
|
752
|
+
|
|
753
|
+
lines.append(" dependencies: []")
|
|
754
|
+
lines.append("```")
|
|
755
|
+
lines.append("")
|
|
756
|
+
|
|
757
|
+
return lines
|
|
758
|
+
|
|
759
|
+
def _get_type_from_schema(self, spec: dict[str, Any]) -> str:
|
|
760
|
+
"""Extract type string from JSON schema property.
|
|
761
|
+
|
|
762
|
+
Parameters
|
|
763
|
+
----------
|
|
764
|
+
spec : dict[str, Any]
|
|
765
|
+
Property specification
|
|
766
|
+
|
|
767
|
+
Returns
|
|
768
|
+
-------
|
|
769
|
+
str
|
|
770
|
+
Human-readable type string
|
|
771
|
+
"""
|
|
772
|
+
if "const" in spec:
|
|
773
|
+
return f'`"{spec["const"]}"`'
|
|
774
|
+
|
|
775
|
+
if "enum" in spec:
|
|
776
|
+
return " | ".join(f'`"{v}`"' for v in spec["enum"][:3])
|
|
777
|
+
|
|
778
|
+
if "anyOf" in spec:
|
|
779
|
+
types = [self._get_type_from_schema(s) for s in spec["anyOf"][:2]]
|
|
780
|
+
return " | ".join(types)
|
|
781
|
+
|
|
782
|
+
type_val = spec.get("type")
|
|
783
|
+
if isinstance(type_val, list):
|
|
784
|
+
return " | ".join(type_val)
|
|
785
|
+
if type_val == "array":
|
|
786
|
+
items = spec.get("items", {})
|
|
787
|
+
item_type = items.get("type", "any")
|
|
788
|
+
return f"list[{item_type}]"
|
|
789
|
+
if type_val:
|
|
790
|
+
return type_val
|
|
791
|
+
|
|
792
|
+
return "any"
|
|
793
|
+
|
|
794
|
+
def _generate_ports_section(self, schema: dict[str, Any]) -> list[str]:
|
|
795
|
+
"""Generate ports documentation from schema.
|
|
796
|
+
|
|
797
|
+
Parameters
|
|
798
|
+
----------
|
|
799
|
+
schema : dict[str, Any]
|
|
800
|
+
Parsed JSON schema
|
|
801
|
+
|
|
802
|
+
Returns
|
|
803
|
+
-------
|
|
804
|
+
list[str]
|
|
805
|
+
Lines of markdown documentation
|
|
806
|
+
"""
|
|
807
|
+
return [
|
|
808
|
+
"## Ports Configuration",
|
|
809
|
+
"",
|
|
810
|
+
"Ports connect pipelines to external services:",
|
|
811
|
+
"",
|
|
812
|
+
"```yaml",
|
|
813
|
+
"spec:",
|
|
814
|
+
" ports:",
|
|
815
|
+
" llm:",
|
|
816
|
+
" adapter: hexdag.builtin.adapters.openai.OpenAIAdapter",
|
|
817
|
+
" config:",
|
|
818
|
+
" api_key: ${OPENAI_API_KEY}",
|
|
819
|
+
" model: gpt-4",
|
|
820
|
+
" memory:",
|
|
821
|
+
" adapter: hexdag.builtin.adapters.memory.InMemoryMemory",
|
|
822
|
+
" database:",
|
|
823
|
+
" adapter: hexdag.builtin.adapters.database.sqlite.SQLiteAdapter",
|
|
824
|
+
" config:",
|
|
825
|
+
" db_path: ./data.db",
|
|
826
|
+
"```",
|
|
827
|
+
"",
|
|
828
|
+
"### Available Port Types",
|
|
829
|
+
"",
|
|
830
|
+
"| Port | Purpose |",
|
|
831
|
+
"|------|---------|",
|
|
832
|
+
"| `llm` | Language model interactions |",
|
|
833
|
+
"| `memory` | Persistent agent memory |",
|
|
834
|
+
"| `database` | Data persistence |",
|
|
835
|
+
"| `secret` | Secret/credential management |",
|
|
836
|
+
"| `tool_router` | Tool invocation routing |",
|
|
837
|
+
"",
|
|
838
|
+
]
|
|
839
|
+
|
|
840
|
+
def _generate_events_section(self, schema: dict[str, Any]) -> list[str]:
|
|
841
|
+
"""Generate events documentation from schema.
|
|
842
|
+
|
|
843
|
+
Parameters
|
|
844
|
+
----------
|
|
845
|
+
schema : dict[str, Any]
|
|
846
|
+
Parsed JSON schema
|
|
847
|
+
|
|
848
|
+
Returns
|
|
849
|
+
-------
|
|
850
|
+
list[str]
|
|
851
|
+
Lines of markdown documentation
|
|
852
|
+
"""
|
|
853
|
+
return [
|
|
854
|
+
"## Events Configuration",
|
|
855
|
+
"",
|
|
856
|
+
"Configure event handlers for observability:",
|
|
857
|
+
"",
|
|
858
|
+
"```yaml",
|
|
859
|
+
"spec:",
|
|
860
|
+
" events:",
|
|
861
|
+
" node_failed:",
|
|
862
|
+
" - type: alert",
|
|
863
|
+
" target: pagerduty",
|
|
864
|
+
" severity: high",
|
|
865
|
+
" pipeline_completed:",
|
|
866
|
+
" - type: metrics",
|
|
867
|
+
" target: datadog",
|
|
868
|
+
"```",
|
|
869
|
+
"",
|
|
870
|
+
"### Event Types",
|
|
871
|
+
"",
|
|
872
|
+
"| Event | When Triggered |",
|
|
873
|
+
"|-------|----------------|",
|
|
874
|
+
"| `pipeline_started` | Pipeline execution begins |",
|
|
875
|
+
"| `pipeline_completed` | Pipeline execution finishes |",
|
|
876
|
+
"| `node_started` | Node execution begins |",
|
|
877
|
+
"| `node_completed` | Node execution finishes |",
|
|
878
|
+
"| `node_failed` | Node execution fails |",
|
|
879
|
+
"",
|
|
880
|
+
"### Handler Types",
|
|
881
|
+
"",
|
|
882
|
+
"| Type | Purpose |",
|
|
883
|
+
"|------|---------|",
|
|
884
|
+
"| `alert` | Send alerts (PagerDuty, Slack) |",
|
|
885
|
+
"| `metrics` | Emit metrics (Datadog, Prometheus) |",
|
|
886
|
+
"| `log` | Write to logs |",
|
|
887
|
+
"| `webhook` | Call external webhooks |",
|
|
888
|
+
"| `callback` | Execute Python callbacks |",
|
|
889
|
+
"",
|
|
890
|
+
]
|